Total coverage: 73712 (4%)of 1851185 






























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * Copyright (c) 2015 Tom Herbert <tom@herbertland.com>
 */

#ifndef __ILA_H
#define __ILA_H

#include <linux/errno.h>
#include <linux/ip.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/socket.h>
#include <linux/skbuff.h>
#include <linux/types.h>
#include <net/checksum.h>
#include <net/genetlink.h>
#include <net/ip.h>
#include <net/protocol.h>
#include <uapi/linux/ila.h>

struct ila_locator {
        union {
                __u8            v8[8];
                __be16          v16[4];
                __be32          v32[2];
                __be64                v64;
        };
};

struct ila_identifier {
        union {
                struct {
#if defined(__LITTLE_ENDIAN_BITFIELD)
                        u8 __space:4;
                        u8 csum_neutral:1;
                        u8 type:3;
#elif defined(__BIG_ENDIAN_BITFIELD)
                        u8 type:3;
                        u8 csum_neutral:1;
                        u8 __space:4;
#else
#error  "Adjust your <asm/byteorder.h> defines"
#endif
                        u8 __space2[7];
                };
                __u8            v8[8];
                __be16          v16[4];
                __be32          v32[2];
                __be64                v64;
        };
};

#define CSUM_NEUTRAL_FLAG        htonl(0x10000000)

struct ila_addr {
        union {
                struct in6_addr addr;
                struct {
                        struct ila_locator loc;
                        struct ila_identifier ident;
                };
        };
};

static inline struct ila_addr *ila_a2i(struct in6_addr *addr)
{
        return (struct ila_addr *)addr;
}

struct ila_params {
        struct ila_locator locator;
        struct ila_locator locator_match;
        __wsum csum_diff;
        u8 csum_mode;
        u8 ident_type;
};

static inline __wsum compute_csum_diff8(const __be32 *from, const __be32 *to)
{
        __be32 diff[] = {
                ~from[0], ~from[1], to[0], to[1],
        };

        return csum_partial(diff, sizeof(diff), 0);
}

static inline bool ila_csum_neutral_set(struct ila_identifier ident)
{
        return !!(ident.csum_neutral);
}

void ila_update_ipv6_locator(struct sk_buff *skb, struct ila_params *p,
                             bool set_csum_neutral);

void ila_init_saved_csum(struct ila_params *p);

struct ila_net {
        struct {
                struct rhashtable rhash_table;
                spinlock_t *locks; /* Bucket locks for entry manipulation */
                unsigned int locks_mask;
                bool hooks_registered;
        } xlat;
};

int ila_lwt_init(void);
void ila_lwt_fini(void);

int ila_xlat_init_net(struct net *net);
void ila_xlat_pre_exit_net(struct net *net);
void ila_xlat_exit_net(struct net *net);

int ila_xlat_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info);
int ila_xlat_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info);
int ila_xlat_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info);
int ila_xlat_nl_cmd_flush(struct sk_buff *skb, struct genl_info *info);
int ila_xlat_nl_dump_start(struct netlink_callback *cb);
int ila_xlat_nl_dump_done(struct netlink_callback *cb);
int ila_xlat_nl_dump(struct sk_buff *skb, struct netlink_callback *cb);

extern unsigned int ila_net_id;

extern struct genl_family ila_nl_family;

#endif /* __ILA_H */

































    5 













    7 






    7 


    5 
    5 






    5 

    5 

































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        SNAP data link layer. Derived from 802.2
 *
 *                Alan Cox <alan@lxorguk.ukuu.org.uk>,
 *                from the 802.2 layer by Greg Page.
 *                Merged in additions from Greg Page's psnap.c.
 */

#include <linux/module.h>
#include <linux/netdevice.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
#include <net/datalink.h>
#include <net/llc.h>
#include <net/psnap.h>
#include <linux/mm.h>
#include <linux/in.h>
#include <linux/init.h>
#include <linux/rculist.h>

static LIST_HEAD(snap_list);
static DEFINE_SPINLOCK(snap_lock);
static struct llc_sap *snap_sap;

/*
 *        Find a snap client by matching the 5 bytes.
 */
static struct datalink_proto *find_snap_client(const unsigned char *desc)
{
        struct datalink_proto *proto = NULL, *p;

        list_for_each_entry_rcu(p, &snap_list, node, lockdep_is_held(&snap_lock)) {
                if (!memcmp(p->type, desc, 5)) {
                        proto = p;
                        break;
                }
        }
        return proto;
}

/*
 *        A SNAP packet has arrived
 */
static int snap_rcv(struct sk_buff *skb, struct net_device *dev,
                    struct packet_type *pt, struct net_device *orig_dev)
{
        int rc = 1;
        struct datalink_proto *proto;
        static struct packet_type snap_packet_type = {
                .type = cpu_to_be16(ETH_P_SNAP),
        };

        if (unlikely(!pskb_may_pull(skb, 5)))
                goto drop;

        rcu_read_lock();
        proto = find_snap_client(skb->data);
        if (proto) {
                /* Pass the frame on. */
                skb_pull_rcsum(skb, 5);
                skb_reset_transport_header(skb);
                rc = proto->rcvfunc(skb, dev, &snap_packet_type, orig_dev);
        }
        rcu_read_unlock();

        if (unlikely(!proto))
                goto drop;

out:
        return rc;

drop:
        kfree_skb(skb);
        goto out;
}

/*
 *        Put a SNAP header on a frame and pass to 802.2
 */
static int snap_request(struct datalink_proto *dl,
                        struct sk_buff *skb, const u8 *dest)
{
        memcpy(skb_push(skb, 5), dl->type, 5);
        llc_build_and_send_ui_pkt(snap_sap, skb, dest, snap_sap->laddr.lsap);
        return 0;
}

/*
 *        Set up the SNAP layer
 */
EXPORT_SYMBOL(register_snap_client);
EXPORT_SYMBOL(unregister_snap_client);

static const char snap_err_msg[] __initconst =
        KERN_CRIT "SNAP - unable to register with 802.2\n";

static int __init snap_init(void)
{
        snap_sap = llc_sap_open(0xAA, snap_rcv);
        if (!snap_sap) {
                printk(snap_err_msg);
                return -EBUSY;
        }

        return 0;
}

module_init(snap_init);

static void __exit snap_exit(void)
{
        llc_sap_put(snap_sap);
}

module_exit(snap_exit);


/*
 *        Register SNAP clients. We don't yet use this for IP.
 */
struct datalink_proto *register_snap_client(const unsigned char *desc,
                                            int (*rcvfunc)(struct sk_buff *,
                                                           struct net_device *,
                                                           struct packet_type *,
                                                           struct net_device *))
{
        struct datalink_proto *proto = NULL;

        spin_lock_bh(&snap_lock);

        if (find_snap_client(desc))
                goto out;

        proto = kmalloc(sizeof(*proto), GFP_ATOMIC);
        if (proto) {
                memcpy(proto->type, desc, 5);
                proto->rcvfunc                = rcvfunc;
                proto->header_length        = 5 + 3; /* snap + 802.2 */
                proto->request                = snap_request;
                list_add_rcu(&proto->node, &snap_list);
        }
out:
        spin_unlock_bh(&snap_lock);

        return proto;
}

/*
 *        Unregister SNAP clients. Protocols no longer want to play with us ...
 */
void unregister_snap_client(struct datalink_proto *proto)
{
        spin_lock_bh(&snap_lock);
        list_del_rcu(&proto->node);
        spin_unlock_bh(&snap_lock);

        synchronize_net();

        kfree(proto);
}

MODULE_DESCRIPTION("SNAP data link layer. Derived from 802.2");
MODULE_LICENSE("GPL");

























































    4 























    4 


    4 







































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
// SPDX-License-Identifier: GPL-2.0-only
/*
 * AppArmor security module
 *
 * This file contains AppArmor security identifier (secid) manipulation fns
 *
 * Copyright 2009-2017 Canonical Ltd.
 *
 * AppArmor allocates a unique secid for every label used. If a label
 * is replaced it receives the secid of the label it is replacing.
 */

#include <linux/errno.h>
#include <linux/err.h>
#include <linux/gfp.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/xarray.h>

#include "include/cred.h"
#include "include/lib.h"
#include "include/secid.h"
#include "include/label.h"
#include "include/policy_ns.h"

/*
 * secids - do not pin labels with a refcount. They rely on the label
 * properly updating/freeing them
 */
#define AA_FIRST_SECID 2

static DEFINE_XARRAY_FLAGS(aa_secids, XA_FLAGS_LOCK_IRQ | XA_FLAGS_TRACK_FREE);

int apparmor_display_secid_mode;

/*
 * TODO: allow policy to reserve a secid range?
 * TODO: add secid pinning
 * TODO: use secid_update in label replace
 */

/*
 * see label for inverse aa_label_to_secid
 */
struct aa_label *aa_secid_to_label(u32 secid)
{
        return xa_load(&aa_secids, secid);
}

static int apparmor_label_to_secctx(struct aa_label *label,
                                    struct lsm_context *cp)
{
        /* TODO: cache secctx and ref count so we don't have to recreate */
        int flags = FLAG_VIEW_SUBNS | FLAG_HIDDEN_UNCONFINED | FLAG_ABS_ROOT;
        int len;

        if (!label)
                return -EINVAL;

        if (apparmor_display_secid_mode)
                flags |= FLAG_SHOW_MODE;

        if (cp)
                len = aa_label_asxprint(&cp->context, root_ns, label,
                                        flags, GFP_ATOMIC);
        else
                len = aa_label_snxprint(NULL, 0, root_ns, label, flags);

        if (len < 0)
                return -ENOMEM;

        if (cp) {
                cp->len = len;
                cp->id = LSM_ID_APPARMOR;
        }

        return len;
}

int apparmor_secid_to_secctx(u32 secid, struct lsm_context *cp)
{
        struct aa_label *label = aa_secid_to_label(secid);

        return apparmor_label_to_secctx(label, cp);
}

int apparmor_lsmprop_to_secctx(struct lsm_prop *prop, struct lsm_context *cp)
{
        struct aa_label *label;

        label = prop->apparmor.label;

        return apparmor_label_to_secctx(label, cp);
}

int apparmor_secctx_to_secid(const char *secdata, u32 seclen, u32 *secid)
{
        struct aa_label *label;

        label = aa_label_strn_parse(&root_ns->unconfined->label, secdata,
                                    seclen, GFP_KERNEL, false, false);
        if (IS_ERR(label))
                return PTR_ERR(label);
        *secid = label->secid;

        return 0;
}

void apparmor_release_secctx(struct lsm_context *cp)
{
        if (cp->id == LSM_ID_APPARMOR) {
                kfree(cp->context);
                cp->context = NULL;
                cp->id = LSM_ID_UNDEF;
        }
}

/**
 * aa_alloc_secid - allocate a new secid for a profile
 * @label: the label to allocate a secid for
 * @gfp: memory allocation flags
 *
 * Returns: 0 with @label->secid initialized
 *          <0 returns error with @label->secid set to AA_SECID_INVALID
 */
int aa_alloc_secid(struct aa_label *label, gfp_t gfp)
{
        unsigned long flags;
        int ret;

        xa_lock_irqsave(&aa_secids, flags);
        ret = __xa_alloc(&aa_secids, &label->secid, label,
                        XA_LIMIT(AA_FIRST_SECID, INT_MAX), gfp);
        xa_unlock_irqrestore(&aa_secids, flags);

        if (ret < 0) {
                label->secid = AA_SECID_INVALID;
                return ret;
        }

        return 0;
}

/**
 * aa_free_secid - free a secid
 * @secid: secid to free
 */
void aa_free_secid(u32 secid)
{
        unsigned long flags;

        xa_lock_irqsave(&aa_secids, flags);
        __xa_erase(&aa_secids, secid);
        xa_unlock_irqrestore(&aa_secids, flags);
}


















    2 













































    5 




    1 

    2 


    2 




    2 





















    2 














    8 







    7 


    8 














   15 




    4 



    2 
   10 

   12 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Cloudflare Ltd https://cloudflare.com */

#include <linux/skmsg.h>
#include <net/sock.h>
#include <net/udp.h>
#include <net/inet_common.h>

#include "udp_impl.h"

static struct proto *udpv6_prot_saved __read_mostly;

static int sk_udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
                          int flags, int *addr_len)
{
#if IS_ENABLED(CONFIG_IPV6)
        if (sk->sk_family == AF_INET6)
                return udpv6_prot_saved->recvmsg(sk, msg, len, flags, addr_len);
#endif
        return udp_prot.recvmsg(sk, msg, len, flags, addr_len);
}

static bool udp_sk_has_data(struct sock *sk)
{
        return !skb_queue_empty(&udp_sk(sk)->reader_queue) ||
               !skb_queue_empty(&sk->sk_receive_queue);
}

static bool psock_has_data(struct sk_psock *psock)
{
        return !skb_queue_empty(&psock->ingress_skb) ||
               !sk_psock_queue_empty(psock);
}

#define udp_msg_has_data(__sk, __psock)        \
                ({ udp_sk_has_data(__sk) || psock_has_data(__psock); })

static int udp_msg_wait_data(struct sock *sk, struct sk_psock *psock,
                             long timeo)
{
        DEFINE_WAIT_FUNC(wait, woken_wake_function);
        int ret = 0;

        if (sk->sk_shutdown & RCV_SHUTDOWN)
                return 1;

        if (!timeo)
                return ret;

        add_wait_queue(sk_sleep(sk), &wait);
        sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
        ret = udp_msg_has_data(sk, psock);
        if (!ret) {
                wait_woken(&wait, TASK_INTERRUPTIBLE, timeo);
                ret = udp_msg_has_data(sk, psock);
        }
        sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
        remove_wait_queue(sk_sleep(sk), &wait);
        return ret;
}

static int udp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
                           int flags, int *addr_len)
{
        struct sk_psock *psock;
        int copied, ret;

        if (unlikely(flags & MSG_ERRQUEUE))
                return inet_recv_error(sk, msg, len, addr_len);

        if (!len)
                return 0;

        psock = sk_psock_get(sk);
        if (unlikely(!psock))
                return sk_udp_recvmsg(sk, msg, len, flags, addr_len);

        if (!psock_has_data(psock)) {
                ret = sk_udp_recvmsg(sk, msg, len, flags, addr_len);
                goto out;
        }

msg_bytes_ready:
        copied = sk_msg_recvmsg(sk, psock, msg, len, flags);
        if (!copied) {
                long timeo;
                int data;

                timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
                data = udp_msg_wait_data(sk, psock, timeo);
                if (data) {
                        if (psock_has_data(psock))
                                goto msg_bytes_ready;
                        ret = sk_udp_recvmsg(sk, msg, len, flags, addr_len);
                        goto out;
                }
                copied = -EAGAIN;
        }
        ret = copied;
out:
        sk_psock_put(sk, psock);
        return ret;
}

enum {
        UDP_BPF_IPV4,
        UDP_BPF_IPV6,
        UDP_BPF_NUM_PROTS,
};

static DEFINE_SPINLOCK(udpv6_prot_lock);
static struct proto udp_bpf_prots[UDP_BPF_NUM_PROTS];

static void udp_bpf_rebuild_protos(struct proto *prot, const struct proto *base)
{
        *prot        = *base;
        prot->close  = sock_map_close;
        prot->recvmsg = udp_bpf_recvmsg;
        prot->sock_is_readable = sk_msg_is_readable;
}

static void udp_bpf_check_v6_needs_rebuild(struct proto *ops)
{
        if (unlikely(ops != smp_load_acquire(&udpv6_prot_saved))) {
                spin_lock_bh(&udpv6_prot_lock);
                if (likely(ops != udpv6_prot_saved)) {
                        udp_bpf_rebuild_protos(&udp_bpf_prots[UDP_BPF_IPV6], ops);
                        smp_store_release(&udpv6_prot_saved, ops);
                }
                spin_unlock_bh(&udpv6_prot_lock);
        }
}

static int __init udp_bpf_v4_build_proto(void)
{
        udp_bpf_rebuild_protos(&udp_bpf_prots[UDP_BPF_IPV4], &udp_prot);
        return 0;
}
late_initcall(udp_bpf_v4_build_proto);

int udp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore)
{
        int family = sk->sk_family == AF_INET ? UDP_BPF_IPV4 : UDP_BPF_IPV6;

        if (restore) {
                sk->sk_write_space = psock->saved_write_space;
                sock_replace_proto(sk, psock->sk_proto);
                return 0;
        }

        if (sk->sk_family == AF_INET6)
                udp_bpf_check_v6_needs_rebuild(psock->sk_proto);

        sock_replace_proto(sk, &udp_bpf_prots[family]);
        return 0;
}
EXPORT_SYMBOL_GPL(udp_bpf_update_proto);
































































    6 































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * File: pep.h
 *
 * Phonet Pipe End Point sockets definitions
 *
 * Copyright (C) 2008 Nokia Corporation.
 */

#ifndef NET_PHONET_PEP_H
#define NET_PHONET_PEP_H

#include <linux/skbuff.h>
#include <net/phonet/phonet.h>

struct pep_sock {
        struct pn_sock                pn_sk;

        /* XXX: union-ify listening vs connected stuff ? */
        /* Listening socket stuff: */
        struct hlist_head        hlist;

        /* Connected socket stuff: */
        struct sock                *listener;
        struct sk_buff_head        ctrlreq_queue;
#define PNPIPE_CTRLREQ_MAX        10
        atomic_t                tx_credits;
        int                        ifindex;
        u16                        peer_type;        /* peer type/subtype */
        u8                        pipe_handle;

        u8                        rx_credits;
        u8                        rx_fc;        /* RX flow control */
        u8                        tx_fc;        /* TX flow control */
        u8                        init_enable;        /* auto-enable at creation */
        u8                        aligned;
};

static inline struct pep_sock *pep_sk(struct sock *sk)
{
        return (struct pep_sock *)sk;
}

extern const struct proto_ops phonet_stream_ops;

/* Pipe protocol definitions */
struct pnpipehdr {
        u8                        utid; /* transaction ID */
        u8                        message_id;
        u8                        pipe_handle;
        union {
                u8                state_after_connect;        /* connect request */
                u8                state_after_reset;        /* reset request */
                u8                error_code;                /* any response */
                u8                pep_type;                /* status indication */
                u8                data0;                        /* anything else */
        };
        u8                        data[];
};
#define other_pep_type                data[0]

static inline struct pnpipehdr *pnp_hdr(struct sk_buff *skb)
{
        return (struct pnpipehdr *)skb_transport_header(skb);
}

#define MAX_PNPIPE_HEADER (MAX_PHONET_HEADER + 4)

enum {
        PNS_PIPE_CREATE_REQ = 0x00,
        PNS_PIPE_CREATE_RESP,
        PNS_PIPE_REMOVE_REQ,
        PNS_PIPE_REMOVE_RESP,

        PNS_PIPE_DATA = 0x20,
        PNS_PIPE_ALIGNED_DATA,

        PNS_PEP_CONNECT_REQ = 0x40,
        PNS_PEP_CONNECT_RESP,
        PNS_PEP_DISCONNECT_REQ,
        PNS_PEP_DISCONNECT_RESP,
        PNS_PEP_RESET_REQ,
        PNS_PEP_RESET_RESP,
        PNS_PEP_ENABLE_REQ,
        PNS_PEP_ENABLE_RESP,
        PNS_PEP_CTRL_REQ,
        PNS_PEP_CTRL_RESP,
        PNS_PEP_DISABLE_REQ = 0x4C,
        PNS_PEP_DISABLE_RESP,

        PNS_PEP_STATUS_IND = 0x60,
        PNS_PIPE_CREATED_IND,
        PNS_PIPE_RESET_IND = 0x63,
        PNS_PIPE_ENABLED_IND,
        PNS_PIPE_REDIRECTED_IND,
        PNS_PIPE_DISABLED_IND = 0x66,
};

#define PN_PIPE_INVALID_HANDLE        0xff
#define PN_PEP_TYPE_COMMON        0x00

/* Phonet pipe status indication */
enum {
        PN_PEP_IND_FLOW_CONTROL,
        PN_PEP_IND_ID_MCFC_GRANT_CREDITS,
};

/* Phonet pipe error codes */
enum {
        PN_PIPE_NO_ERROR,
        PN_PIPE_ERR_INVALID_PARAM,
        PN_PIPE_ERR_INVALID_HANDLE,
        PN_PIPE_ERR_INVALID_CTRL_ID,
        PN_PIPE_ERR_NOT_ALLOWED,
        PN_PIPE_ERR_PEP_IN_USE,
        PN_PIPE_ERR_OVERLOAD,
        PN_PIPE_ERR_DEV_DISCONNECTED,
        PN_PIPE_ERR_TIMEOUT,
        PN_PIPE_ERR_ALL_PIPES_IN_USE,
        PN_PIPE_ERR_GENERAL,
        PN_PIPE_ERR_NOT_SUPPORTED,
};

/* Phonet pipe states */
enum {
        PN_PIPE_DISABLE,
        PN_PIPE_ENABLE,
};

/* Phonet pipe sub-block types */
enum {
        PN_PIPE_SB_CREATE_REQ_PEP_SUB_TYPE,
        PN_PIPE_SB_CONNECT_REQ_PEP_SUB_TYPE,
        PN_PIPE_SB_REDIRECT_REQ_PEP_SUB_TYPE,
        PN_PIPE_SB_NEGOTIATED_FC,
        PN_PIPE_SB_REQUIRED_FC_TX,
        PN_PIPE_SB_PREFERRED_FC_RX,
        PN_PIPE_SB_ALIGNED_DATA,
};

/* Phonet pipe flow control models */
enum {
        PN_NO_FLOW_CONTROL,
        PN_LEGACY_FLOW_CONTROL,
        PN_ONE_CREDIT_FLOW_CONTROL,
        PN_MULTI_CREDIT_FLOW_CONTROL,
        PN_MAX_FLOW_CONTROL,
};

#define pn_flow_safe(fc) ((fc) >> 1)

/* Phonet pipe flow control states */
enum {
        PEP_IND_EMPTY,
        PEP_IND_BUSY,
        PEP_IND_READY,
};

#endif





























































































   15 




    1 






    4 




   12 























   15 
    6 



















    8 




    8 

    3 
    5 




























































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
/*
 * Routines to compress and uncompress tcp packets (for transmission
 * over low speed serial lines).
 *
 * Copyright (c) 1989 Regents of the University of California.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms are permitted
 * provided that the above copyright notice and this paragraph are
 * duplicated in all such forms and that any documentation,
 * advertising materials, and other materials related to such
 * distribution and use acknowledge that the software was developed
 * by the University of California, Berkeley.  The name of the
 * University may not be used to endorse or promote products derived
 * from this software without specific prior written permission.
 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 *
 *        Van Jacobson (van@helios.ee.lbl.gov), Dec 31, 1989:
 *        - Initial distribution.
 *
 *
 * modified for KA9Q Internet Software Package by
 * Katie Stevens (dkstevens@ucdavis.edu)
 * University of California, Davis
 * Computing Services
 *        - 01-31-90        initial adaptation (from 1.19)
 *        PPP.05        02-15-90 [ks]
 *        PPP.08        05-02-90 [ks]        use PPP protocol field to signal compression
 *        PPP.15        09-90         [ks]        improve mbuf handling
 *        PPP.16        11-02         [karn]        substantially rewritten to use NOS facilities
 *
 *        - Feb 1991        Bill_Simpson@um.cc.umich.edu
 *                        variable number of conversation slots
 *                        allow zero or one slots
 *                        separate routines
 *                        status display
 *        - Jul 1994        Dmitry Gorodchanin
 *                        Fixes for memory leaks.
 *      - Oct 1994      Dmitry Gorodchanin
 *                      Modularization.
 *        - Jan 1995        Bjorn Ekwall
 *                        Use ip_fast_csum from ip.h
 *        - July 1995        Christos A. Polyzols
 *                        Spotted bug in tcp option checking
 *
 *
 *        This module is a difficult issue. It's clearly inet code but it's also clearly
 *        driver code belonging close to PPP and SLIP
 */

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <net/slhc_vj.h>

#ifdef CONFIG_INET
/* Entire module is for IP only */
#include <linux/mm.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/termios.h>
#include <linux/in.h>
#include <linux/fcntl.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <net/ip.h>
#include <net/protocol.h>
#include <net/icmp.h>
#include <net/tcp.h>
#include <linux/skbuff.h>
#include <net/sock.h>
#include <linux/timer.h>
#include <linux/uaccess.h>
#include <net/checksum.h>
#include <linux/unaligned.h>

static unsigned char *encode(unsigned char *cp, unsigned short n);
static long decode(unsigned char **cpp);
static unsigned char * put16(unsigned char *cp, unsigned short x);
static unsigned short pull16(unsigned char **cpp);

/* Allocate compression data structure
 *        slots must be in range 0 to 255 (zero meaning no compression)
 * Returns pointer to structure or ERR_PTR() on error.
 */
struct slcompress *
slhc_init(int rslots, int tslots)
{
        short i;
        struct cstate *ts;
        struct slcompress *comp;

        if (rslots < 0 || rslots > 255 || tslots < 0 || tslots > 255)
                return ERR_PTR(-EINVAL);

        comp = kzalloc(sizeof(struct slcompress), GFP_KERNEL);
        if (! comp)
                goto out_fail;

        if (rslots > 0) {
                size_t rsize = rslots * sizeof(struct cstate);
                comp->rstate = kzalloc(rsize, GFP_KERNEL);
                if (! comp->rstate)
                        goto out_free;
                comp->rslot_limit = rslots - 1;
        }

        if (tslots > 0) {
                size_t tsize = tslots * sizeof(struct cstate);
                comp->tstate = kzalloc(tsize, GFP_KERNEL);
                if (! comp->tstate)
                        goto out_free2;
                comp->tslot_limit = tslots - 1;
        }

        comp->xmit_oldest = 0;
        comp->xmit_current = 255;
        comp->recv_current = 255;
        /*
         * don't accept any packets with implicit index until we get
         * one with an explicit index.  Otherwise the uncompress code
         * will try to use connection 255, which is almost certainly
         * out of range
         */
        comp->flags |= SLF_TOSS;

        if ( tslots > 0 ) {
                ts = comp->tstate;
                for(i = comp->tslot_limit; i > 0; --i){
                        ts[i].cs_this = i;
                        ts[i].next = &(ts[i - 1]);
                }
                ts[0].next = &(ts[comp->tslot_limit]);
                ts[0].cs_this = 0;
        }
        return comp;

out_free2:
        kfree(comp->rstate);
out_free:
        kfree(comp);
out_fail:
        return ERR_PTR(-ENOMEM);
}


/* Free a compression data structure */
void
slhc_free(struct slcompress *comp)
{
        if ( IS_ERR_OR_NULL(comp) )
                return;

        if ( comp->tstate != NULLSLSTATE )
                kfree( comp->tstate );

        if ( comp->rstate != NULLSLSTATE )
                kfree( comp->rstate );

        kfree( comp );
}


/* Put a short in host order into a char array in network order */
static inline unsigned char *
put16(unsigned char *cp, unsigned short x)
{
        *cp++ = x >> 8;
        *cp++ = x;

        return cp;
}


/* Encode a number */
static unsigned char *
encode(unsigned char *cp, unsigned short n)
{
        if(n >= 256 || n == 0){
                *cp++ = 0;
                cp = put16(cp,n);
        } else {
                *cp++ = n;
        }
        return cp;
}

/* Pull a 16-bit integer in host order from buffer in network byte order */
static unsigned short
pull16(unsigned char **cpp)
{
        short rval;

        rval = *(*cpp)++;
        rval <<= 8;
        rval |= *(*cpp)++;
        return rval;
}

/* Decode a number */
static long
decode(unsigned char **cpp)
{
        int x;

        x = *(*cpp)++;
        if(x == 0){
                return pull16(cpp) & 0xffff;        /* pull16 returns -1 on error */
        } else {
                return x & 0xff;                /* -1 if PULLCHAR returned error */
        }
}

/*
 * icp and isize are the original packet.
 * ocp is a place to put a copy if necessary.
 * cpp is initially a pointer to icp.  If the copy is used,
 *    change it to ocp.
 */

int
slhc_compress(struct slcompress *comp, unsigned char *icp, int isize,
        unsigned char *ocp, unsigned char **cpp, int compress_cid)
{
        struct cstate *ocs = &(comp->tstate[comp->xmit_oldest]);
        struct cstate *lcs = ocs;
        struct cstate *cs = lcs->next;
        unsigned long deltaS, deltaA;
        short changes = 0;
        int nlen, hlen;
        unsigned char new_seq[16];
        unsigned char *cp = new_seq;
        struct iphdr *ip;
        struct tcphdr *th, *oth;
        __sum16 csum;


        /*
         *        Don't play with runt packets.
         */

        if(isize<sizeof(struct iphdr))
                return isize;

        ip = (struct iphdr *) icp;
        if (ip->version != 4 || ip->ihl < 5)
                return isize;

        /* Bail if this packet isn't TCP, or is an IP fragment */
        if (ip->protocol != IPPROTO_TCP || (ntohs(ip->frag_off) & 0x3fff)) {
                /* Send as regular IP */
                if(ip->protocol != IPPROTO_TCP)
                        comp->sls_o_nontcp++;
                else
                        comp->sls_o_tcp++;
                return isize;
        }
        nlen = ip->ihl * 4;
        if (isize < nlen + sizeof(*th))
                return isize;

        th = (struct tcphdr *)(icp + nlen);
        if (th->doff < sizeof(struct tcphdr) / 4)
                return isize;
        hlen = nlen + th->doff * 4;

        /*  Bail if the TCP packet isn't `compressible' (i.e., ACK isn't set or
         *  some other control bit is set). Also uncompressible if
         *  it's a runt.
         */
        if(hlen > isize || th->syn || th->fin || th->rst ||
            ! (th->ack)){
                /* TCP connection stuff; send as regular IP */
                comp->sls_o_tcp++;
                return isize;
        }
        /*
         * Packet is compressible -- we're going to send either a
         * COMPRESSED_TCP or UNCOMPRESSED_TCP packet.  Either way,
         * we need to locate (or create) the connection state.
         *
         * States are kept in a circularly linked list with
         * xmit_oldest pointing to the end of the list.  The
         * list is kept in lru order by moving a state to the
         * head of the list whenever it is referenced.  Since
         * the list is short and, empirically, the connection
         * we want is almost always near the front, we locate
         * states via linear search.  If we don't find a state
         * for the datagram, the oldest state is (re-)used.
         */
        for ( ; ; ) {
                if( ip->saddr == cs->cs_ip.saddr
                 && ip->daddr == cs->cs_ip.daddr
                 && th->source == cs->cs_tcp.source
                 && th->dest == cs->cs_tcp.dest)
                        goto found;

                /* if current equal oldest, at end of list */
                if ( cs == ocs )
                        break;
                lcs = cs;
                cs = cs->next;
                comp->sls_o_searches++;
        }
        /*
         * Didn't find it -- re-use oldest cstate.  Send an
         * uncompressed packet that tells the other side what
         * connection number we're using for this conversation.
         *
         * Note that since the state list is circular, the oldest
         * state points to the newest and we only need to set
         * xmit_oldest to update the lru linkage.
         */
        comp->sls_o_misses++;
        comp->xmit_oldest = lcs->cs_this;
        goto uncompressed;

found:
        /*
         * Found it -- move to the front on the connection list.
         */
        if(lcs == ocs) {
                /* found at most recently used */
        } else if (cs == ocs) {
                /* found at least recently used */
                comp->xmit_oldest = lcs->cs_this;
        } else {
                /* more than 2 elements */
                lcs->next = cs->next;
                cs->next = ocs->next;
                ocs->next = cs;
        }

        /*
         * Make sure that only what we expect to change changed.
         * Check the following:
         * IP protocol version, header length & type of service.
         * The "Don't fragment" bit.
         * The time-to-live field.
         * The TCP header length.
         * IP options, if any.
         * TCP options, if any.
         * If any of these things are different between the previous &
         * current datagram, we send the current datagram `uncompressed'.
         */
        oth = &cs->cs_tcp;

        if(ip->version != cs->cs_ip.version || ip->ihl != cs->cs_ip.ihl
         || ip->tos != cs->cs_ip.tos
         || (ip->frag_off & htons(0x4000)) != (cs->cs_ip.frag_off & htons(0x4000))
         || ip->ttl != cs->cs_ip.ttl
         || th->doff != cs->cs_tcp.doff
         || (ip->ihl > 5 && memcmp(ip+1,cs->cs_ipopt,((ip->ihl)-5)*4) != 0)
         || (th->doff > 5 && memcmp(th+1,cs->cs_tcpopt,((th->doff)-5)*4) != 0)){
                goto uncompressed;
        }

        /*
         * Figure out which of the changing fields changed.  The
         * receiver expects changes in the order: urgent, window,
         * ack, seq (the order minimizes the number of temporaries
         * needed in this section of code).
         */
        if(th->urg){
                deltaS = ntohs(th->urg_ptr);
                cp = encode(cp,deltaS);
                changes |= NEW_U;
        } else if(th->urg_ptr != oth->urg_ptr){
                /* argh! URG not set but urp changed -- a sensible
                 * implementation should never do this but RFC793
                 * doesn't prohibit the change so we have to deal
                 * with it. */
                goto uncompressed;
        }
        if((deltaS = ntohs(th->window) - ntohs(oth->window)) != 0){
                cp = encode(cp,deltaS);
                changes |= NEW_W;
        }
        if((deltaA = ntohl(th->ack_seq) - ntohl(oth->ack_seq)) != 0L){
                if(deltaA > 0x0000ffff)
                        goto uncompressed;
                cp = encode(cp,deltaA);
                changes |= NEW_A;
        }
        if((deltaS = ntohl(th->seq) - ntohl(oth->seq)) != 0L){
                if(deltaS > 0x0000ffff)
                        goto uncompressed;
                cp = encode(cp,deltaS);
                changes |= NEW_S;
        }

        switch(changes){
        case 0:        /* Nothing changed. If this packet contains data and the
                 * last one didn't, this is probably a data packet following
                 * an ack (normal on an interactive connection) and we send
                 * it compressed.  Otherwise it's probably a retransmit,
                 * retransmitted ack or window probe.  Send it uncompressed
                 * in case the other side missed the compressed version.
                 */
                if(ip->tot_len != cs->cs_ip.tot_len &&
                   ntohs(cs->cs_ip.tot_len) == hlen)
                        break;
                goto uncompressed;
        case SPECIAL_I:
        case SPECIAL_D:
                /* actual changes match one of our special case encodings --
                 * send packet uncompressed.
                 */
                goto uncompressed;
        case NEW_S|NEW_A:
                if(deltaS == deltaA &&
                    deltaS == ntohs(cs->cs_ip.tot_len) - hlen){
                        /* special case for echoed terminal traffic */
                        changes = SPECIAL_I;
                        cp = new_seq;
                }
                break;
        case NEW_S:
                if(deltaS == ntohs(cs->cs_ip.tot_len) - hlen){
                        /* special case for data xfer */
                        changes = SPECIAL_D;
                        cp = new_seq;
                }
                break;
        }
        deltaS = ntohs(ip->id) - ntohs(cs->cs_ip.id);
        if(deltaS != 1){
                cp = encode(cp,deltaS);
                changes |= NEW_I;
        }
        if(th->psh)
                changes |= TCP_PUSH_BIT;
        /* Grab the cksum before we overwrite it below.  Then update our
         * state with this packet's header.
         */
        csum = th->check;
        memcpy(&cs->cs_ip,ip,20);
        memcpy(&cs->cs_tcp,th,20);
        /* We want to use the original packet as our compressed packet.
         * (cp - new_seq) is the number of bytes we need for compressed
         * sequence numbers.  In addition we need one byte for the change
         * mask, one for the connection id and two for the tcp checksum.
         * So, (cp - new_seq) + 4 bytes of header are needed.
         */
        deltaS = cp - new_seq;
        if(compress_cid == 0 || comp->xmit_current != cs->cs_this){
                cp = ocp;
                *cpp = ocp;
                *cp++ = changes | NEW_C;
                *cp++ = cs->cs_this;
                comp->xmit_current = cs->cs_this;
        } else {
                cp = ocp;
                *cpp = ocp;
                *cp++ = changes;
        }
        *(__sum16 *)cp = csum;
        cp += 2;
/* deltaS is now the size of the change section of the compressed header */
        memcpy(cp,new_seq,deltaS);        /* Write list of deltas */
        memcpy(cp+deltaS,icp+hlen,isize-hlen);
        comp->sls_o_compressed++;
        ocp[0] |= SL_TYPE_COMPRESSED_TCP;
        return isize - hlen + deltaS + (cp - ocp);

        /* Update connection state cs & send uncompressed packet (i.e.,
         * a regular ip/tcp packet but with the 'conversation id' we hope
         * to use on future compressed packets in the protocol field).
         */
uncompressed:
        memcpy(&cs->cs_ip,ip,20);
        memcpy(&cs->cs_tcp,th,20);
        if (ip->ihl > 5)
          memcpy(cs->cs_ipopt, ip+1, ((ip->ihl) - 5) * 4);
        if (th->doff > 5)
          memcpy(cs->cs_tcpopt, th+1, ((th->doff) - 5) * 4);
        comp->xmit_current = cs->cs_this;
        comp->sls_o_uncompressed++;
        memcpy(ocp, icp, isize);
        *cpp = ocp;
        ocp[9] = cs->cs_this;
        ocp[0] |= SL_TYPE_UNCOMPRESSED_TCP;
        return isize;
}


int
slhc_uncompress(struct slcompress *comp, unsigned char *icp, int isize)
{
        int changes;
        long x;
        struct tcphdr *thp;
        struct iphdr *ip;
        struct cstate *cs;
        int len, hdrlen;
        unsigned char *cp = icp;

        /* We've got a compressed packet; read the change byte */
        comp->sls_i_compressed++;
        if(isize < 3){
                comp->sls_i_error++;
                return 0;
        }
        changes = *cp++;
        if(changes & NEW_C){
                /* Make sure the state index is in range, then grab the state.
                 * If we have a good state index, clear the 'discard' flag.
                 */
                x = *cp++;        /* Read conn index */
                if(x < 0 || x > comp->rslot_limit)
                        goto bad;

                /* Check if the cstate is initialized */
                if (!comp->rstate[x].initialized)
                        goto bad;

                comp->flags &=~ SLF_TOSS;
                comp->recv_current = x;
        } else {
                /* this packet has an implicit state index.  If we've
                 * had a line error since the last time we got an
                 * explicit state index, we have to toss the packet. */
                if(comp->flags & SLF_TOSS){
                        comp->sls_i_tossed++;
                        return 0;
                }
        }
        cs = &comp->rstate[comp->recv_current];
        thp = &cs->cs_tcp;
        ip = &cs->cs_ip;

        thp->check = *(__sum16 *)cp;
        cp += 2;

        thp->psh = (changes & TCP_PUSH_BIT) ? 1 : 0;
/*
 * we can use the same number for the length of the saved header and
 * the current one, because the packet wouldn't have been sent
 * as compressed unless the options were the same as the previous one
 */

        hdrlen = ip->ihl * 4 + thp->doff * 4;

        switch(changes & SPECIALS_MASK){
        case SPECIAL_I:                /* Echoed terminal traffic */
                {
                short i;
                i = ntohs(ip->tot_len) - hdrlen;
                thp->ack_seq = htonl( ntohl(thp->ack_seq) + i);
                thp->seq = htonl( ntohl(thp->seq) + i);
                }
                break;

        case SPECIAL_D:                        /* Unidirectional data */
                thp->seq = htonl( ntohl(thp->seq) +
                                  ntohs(ip->tot_len) - hdrlen);
                break;

        default:
                if(changes & NEW_U){
                        thp->urg = 1;
                        if((x = decode(&cp)) == -1) {
                                goto bad;
                        }
                        thp->urg_ptr = htons(x);
                } else
                        thp->urg = 0;
                if(changes & NEW_W){
                        if((x = decode(&cp)) == -1) {
                                goto bad;
                        }
                        thp->window = htons( ntohs(thp->window) + x);
                }
                if(changes & NEW_A){
                        if((x = decode(&cp)) == -1) {
                                goto bad;
                        }
                        thp->ack_seq = htonl( ntohl(thp->ack_seq) + x);
                }
                if(changes & NEW_S){
                        if((x = decode(&cp)) == -1) {
                                goto bad;
                        }
                        thp->seq = htonl( ntohl(thp->seq) + x);
                }
                break;
        }
        if(changes & NEW_I){
                if((x = decode(&cp)) == -1) {
                        goto bad;
                }
                ip->id = htons (ntohs (ip->id) + x);
        } else
                ip->id = htons (ntohs (ip->id) + 1);

        /*
         * At this point, cp points to the first byte of data in the
         * packet.  Put the reconstructed TCP and IP headers back on the
         * packet.  Recalculate IP checksum (but not TCP checksum).
         */

        len = isize - (cp - icp);
        if (len < 0)
                goto bad;
        len += hdrlen;
        ip->tot_len = htons(len);
        ip->check = 0;

        memmove(icp + hdrlen, cp, len - hdrlen);

        cp = icp;
        memcpy(cp, ip, 20);
        cp += 20;

        if (ip->ihl > 5) {
          memcpy(cp, cs->cs_ipopt, (ip->ihl - 5) * 4);
          cp += (ip->ihl - 5) * 4;
        }

        put_unaligned(ip_fast_csum(icp, ip->ihl),
                      &((struct iphdr *)icp)->check);

        memcpy(cp, thp, 20);
        cp += 20;

        if (thp->doff > 5) {
          memcpy(cp, cs->cs_tcpopt, ((thp->doff) - 5) * 4);
          cp += ((thp->doff) - 5) * 4;
        }

        return len;
bad:
        comp->sls_i_error++;
        return slhc_toss( comp );
}


int
slhc_remember(struct slcompress *comp, unsigned char *icp, int isize)
{
        const struct tcphdr *th;
        unsigned char index;
        struct iphdr *iph;
        struct cstate *cs;
        unsigned int ihl;

        /* The packet is shorter than a legal IP header.
         * Also make sure isize is positive.
         */
        if (isize < (int)sizeof(struct iphdr)) {
runt:
                comp->sls_i_runt++;
                return slhc_toss(comp);
        }
        iph = (struct iphdr *)icp;
        /* Peek at the IP header's IHL field to find its length */
        ihl = iph->ihl;
        /* The IP header length field is too small,
         * or packet is shorter than the IP header followed
         * by minimal tcp header.
         */
        if (ihl < 5 || isize < ihl * 4 + sizeof(struct tcphdr))
                goto runt;

        index = iph->protocol;
        iph->protocol = IPPROTO_TCP;

        if (ip_fast_csum(icp, ihl)) {
                /* Bad IP header checksum; discard */
                comp->sls_i_badcheck++;
                return slhc_toss(comp);
        }
        if (index > comp->rslot_limit) {
                comp->sls_i_error++;
                return slhc_toss(comp);
        }
        th = (struct tcphdr *)(icp + ihl * 4);
        if (th->doff < sizeof(struct tcphdr) / 4)
                goto runt;
        if (isize < ihl * 4 + th->doff * 4)
                goto runt;
        /* Update local state */
        cs = &comp->rstate[comp->recv_current = index];
        comp->flags &=~ SLF_TOSS;
        memcpy(&cs->cs_ip, iph, sizeof(*iph));
        memcpy(&cs->cs_tcp, th, sizeof(*th));
        if (ihl > 5)
          memcpy(cs->cs_ipopt, &iph[1], (ihl - 5) * 4);
        if (th->doff > 5)
          memcpy(cs->cs_tcpopt, &th[1], (th->doff - 5) * 4);
        cs->cs_hsize = ihl*2 + th->doff*2;
        cs->initialized = true;
        /* Put headers back on packet
         * Neither header checksum is recalculated
         */
        comp->sls_i_uncompressed++;
        return isize;
}

int
slhc_toss(struct slcompress *comp)
{
        if ( comp == NULLSLCOMPR )
                return 0;

        comp->flags |= SLF_TOSS;
        return 0;
}

#else /* CONFIG_INET */

int
slhc_toss(struct slcompress *comp)
{
  printk(KERN_DEBUG "Called IP function on non IP-system: slhc_toss");
  return -EINVAL;
}
int
slhc_uncompress(struct slcompress *comp, unsigned char *icp, int isize)
{
  printk(KERN_DEBUG "Called IP function on non IP-system: slhc_uncompress");
  return -EINVAL;
}
int
slhc_compress(struct slcompress *comp, unsigned char *icp, int isize,
        unsigned char *ocp, unsigned char **cpp, int compress_cid)
{
  printk(KERN_DEBUG "Called IP function on non IP-system: slhc_compress");
  return -EINVAL;
}

int
slhc_remember(struct slcompress *comp, unsigned char *icp, int isize)
{
  printk(KERN_DEBUG "Called IP function on non IP-system: slhc_remember");
  return -EINVAL;
}

void
slhc_free(struct slcompress *comp)
{
  printk(KERN_DEBUG "Called IP function on non IP-system: slhc_free");
}
struct slcompress *
slhc_init(int rslots, int tslots)
{
  printk(KERN_DEBUG "Called IP function on non IP-system: slhc_init");
  return NULL;
}

#endif /* CONFIG_INET */

/* VJ header compression */
EXPORT_SYMBOL(slhc_init);
EXPORT_SYMBOL(slhc_free);
EXPORT_SYMBOL(slhc_remember);
EXPORT_SYMBOL(slhc_compress);
EXPORT_SYMBOL(slhc_uncompress);
EXPORT_SYMBOL(slhc_toss);

MODULE_DESCRIPTION("Compression helpers for SLIP (serial line)");
MODULE_LICENSE("Dual BSD/GPL");














































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    9 






























































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_SCHED_H
#define _LINUX_SCHED_H

/*
 * Define 'struct task_struct' and provide the main scheduler
 * APIs (schedule(), wakeup variants, etc.)
 */

#include <uapi/linux/sched.h>

#include <asm/current.h>
#include <asm/processor.h>
#include <linux/thread_info.h>
#include <linux/preempt.h>
#include <linux/cpumask_types.h>

#include <linux/cache.h>
#include <linux/irqflags_types.h>
#include <linux/smp_types.h>
#include <linux/pid_types.h>
#include <linux/sem_types.h>
#include <linux/shm.h>
#include <linux/kmsan_types.h>
#include <linux/mutex_types.h>
#include <linux/plist_types.h>
#include <linux/hrtimer_types.h>
#include <linux/timer_types.h>
#include <linux/seccomp_types.h>
#include <linux/nodemask_types.h>
#include <linux/refcount_types.h>
#include <linux/resource.h>
#include <linux/latencytop.h>
#include <linux/sched/prio.h>
#include <linux/sched/types.h>
#include <linux/signal_types.h>
#include <linux/syscall_user_dispatch_types.h>
#include <linux/mm_types_task.h>
#include <linux/netdevice_xmit.h>
#include <linux/task_io_accounting.h>
#include <linux/posix-timers_types.h>
#include <linux/restart_block.h>
#include <uapi/linux/rseq.h>
#include <linux/seqlock_types.h>
#include <linux/kcsan.h>
#include <linux/rv.h>
#include <linux/livepatch_sched.h>
#include <linux/uidgid_types.h>
#include <linux/tracepoint-defs.h>
#include <asm/kmap_size.h>

/* task_struct member predeclarations (sorted alphabetically): */
struct audit_context;
struct bio_list;
struct blk_plug;
struct bpf_local_storage;
struct bpf_run_ctx;
struct bpf_net_context;
struct capture_control;
struct cfs_rq;
struct fs_struct;
struct futex_pi_state;
struct io_context;
struct io_uring_task;
struct mempolicy;
struct nameidata;
struct nsproxy;
struct perf_event_context;
struct perf_ctx_data;
struct pid_namespace;
struct pipe_inode_info;
struct rcu_node;
struct reclaim_state;
struct robust_list_head;
struct root_domain;
struct rq;
struct sched_attr;
struct sched_dl_entity;
struct seq_file;
struct sighand_struct;
struct signal_struct;
struct task_delay_info;
struct task_group;
struct task_struct;
struct user_event_mm;

#include <linux/sched/ext.h>

/*
 * Task state bitmask. NOTE! These bits are also
 * encoded in fs/proc/array.c: get_task_state().
 *
 * We have two separate sets of flags: task->__state
 * is about runnability, while task->exit_state are
 * about the task exiting. Confusing, but this way
 * modifying one set can't modify the other one by
 * mistake.
 */

/* Used in tsk->__state: */
#define TASK_RUNNING                        0x00000000
#define TASK_INTERRUPTIBLE                0x00000001
#define TASK_UNINTERRUPTIBLE                0x00000002
#define __TASK_STOPPED                        0x00000004
#define __TASK_TRACED                        0x00000008
/* Used in tsk->exit_state: */
#define EXIT_DEAD                        0x00000010
#define EXIT_ZOMBIE                        0x00000020
#define EXIT_TRACE                        (EXIT_ZOMBIE | EXIT_DEAD)
/* Used in tsk->__state again: */
#define TASK_PARKED                        0x00000040
#define TASK_DEAD                        0x00000080
#define TASK_WAKEKILL                        0x00000100
#define TASK_WAKING                        0x00000200
#define TASK_NOLOAD                        0x00000400
#define TASK_NEW                        0x00000800
#define TASK_RTLOCK_WAIT                0x00001000
#define TASK_FREEZABLE                        0x00002000
#define __TASK_FREEZABLE_UNSAFE               (0x00004000 * IS_ENABLED(CONFIG_LOCKDEP))
#define TASK_FROZEN                        0x00008000
#define TASK_STATE_MAX                        0x00010000

#define TASK_ANY                        (TASK_STATE_MAX-1)

/*
 * DO NOT ADD ANY NEW USERS !
 */
#define TASK_FREEZABLE_UNSAFE                (TASK_FREEZABLE | __TASK_FREEZABLE_UNSAFE)

/* Convenience macros for the sake of set_current_state: */
#define TASK_KILLABLE                        (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
#define TASK_STOPPED                        (TASK_WAKEKILL | __TASK_STOPPED)
#define TASK_TRACED                        __TASK_TRACED

#define TASK_IDLE                        (TASK_UNINTERRUPTIBLE | TASK_NOLOAD)

/* Convenience macros for the sake of wake_up(): */
#define TASK_NORMAL                        (TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)

/* get_task_state(): */
#define TASK_REPORT                        (TASK_RUNNING | TASK_INTERRUPTIBLE | \
                                         TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
                                         __TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \
                                         TASK_PARKED)

#define task_is_running(task)                (READ_ONCE((task)->__state) == TASK_RUNNING)

#define task_is_traced(task)                ((READ_ONCE(task->jobctl) & JOBCTL_TRACED) != 0)
#define task_is_stopped(task)                ((READ_ONCE(task->jobctl) & JOBCTL_STOPPED) != 0)
#define task_is_stopped_or_traced(task)        ((READ_ONCE(task->jobctl) & (JOBCTL_STOPPED | JOBCTL_TRACED)) != 0)

/*
 * Special states are those that do not use the normal wait-loop pattern. See
 * the comment with set_special_state().
 */
#define is_special_task_state(state)                                        \
        ((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED |        \
                    TASK_DEAD | TASK_FROZEN))

#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
# define debug_normal_state_change(state_value)                                \
        do {                                                                \
                WARN_ON_ONCE(is_special_task_state(state_value));        \
                current->task_state_change = _THIS_IP_;                        \
        } while (0)

# define debug_special_state_change(state_value)                        \
        do {                                                                \
                WARN_ON_ONCE(!is_special_task_state(state_value));        \
                current->task_state_change = _THIS_IP_;                        \
        } while (0)

# define debug_rtlock_wait_set_state()                                        \
        do {                                                                 \
                current->saved_state_change = current->task_state_change;\
                current->task_state_change = _THIS_IP_;                         \
        } while (0)

# define debug_rtlock_wait_restore_state()                                \
        do {                                                                 \
                current->task_state_change = current->saved_state_change;\
        } while (0)

#else
# define debug_normal_state_change(cond)        do { } while (0)
# define debug_special_state_change(cond)        do { } while (0)
# define debug_rtlock_wait_set_state()                do { } while (0)
# define debug_rtlock_wait_restore_state()        do { } while (0)
#endif

#define trace_set_current_state(state_value)                     \
        do {                                                     \
                if (tracepoint_enabled(sched_set_state_tp))      \
                        __trace_set_current_state(state_value); \
        } while (0)

/*
 * set_current_state() includes a barrier so that the write of current->__state
 * is correctly serialised wrt the caller's subsequent test of whether to
 * actually sleep:
 *
 *   for (;;) {
 *        set_current_state(TASK_UNINTERRUPTIBLE);
 *        if (CONDITION)
 *           break;
 *
 *        schedule();
 *   }
 *   __set_current_state(TASK_RUNNING);
 *
 * If the caller does not need such serialisation (because, for instance, the
 * CONDITION test and condition change and wakeup are under the same lock) then
 * use __set_current_state().
 *
 * The above is typically ordered against the wakeup, which does:
 *
 *   CONDITION = 1;
 *   wake_up_state(p, TASK_UNINTERRUPTIBLE);
 *
 * where wake_up_state()/try_to_wake_up() executes a full memory barrier before
 * accessing p->__state.
 *
 * Wakeup will do: if (@state & p->__state) p->__state = TASK_RUNNING, that is,
 * once it observes the TASK_UNINTERRUPTIBLE store the waking CPU can issue a
 * TASK_RUNNING store which can collide with __set_current_state(TASK_RUNNING).
 *
 * However, with slightly different timing the wakeup TASK_RUNNING store can
 * also collide with the TASK_UNINTERRUPTIBLE store. Losing that store is not
 * a problem either because that will result in one extra go around the loop
 * and our @cond test will save the day.
 *
 * Also see the comments of try_to_wake_up().
 */
#define __set_current_state(state_value)                                \
        do {                                                                \
                debug_normal_state_change((state_value));                \
                trace_set_current_state(state_value);                        \
                WRITE_ONCE(current->__state, (state_value));                \
        } while (0)

#define set_current_state(state_value)                                        \
        do {                                                                \
                debug_normal_state_change((state_value));                \
                trace_set_current_state(state_value);                        \
                smp_store_mb(current->__state, (state_value));                \
        } while (0)

/*
 * set_special_state() should be used for those states when the blocking task
 * can not use the regular condition based wait-loop. In that case we must
 * serialize against wakeups such that any possible in-flight TASK_RUNNING
 * stores will not collide with our state change.
 */
#define set_special_state(state_value)                                        \
        do {                                                                \
                unsigned long flags; /* may shadow */                        \
                                                                        \
                raw_spin_lock_irqsave(&current->pi_lock, flags);        \
                debug_special_state_change((state_value));                \
                trace_set_current_state(state_value);                        \
                WRITE_ONCE(current->__state, (state_value));                \
                raw_spin_unlock_irqrestore(&current->pi_lock, flags);        \
        } while (0)

/*
 * PREEMPT_RT specific variants for "sleeping" spin/rwlocks
 *
 * RT's spin/rwlock substitutions are state preserving. The state of the
 * task when blocking on the lock is saved in task_struct::saved_state and
 * restored after the lock has been acquired.  These operations are
 * serialized by task_struct::pi_lock against try_to_wake_up(). Any non RT
 * lock related wakeups while the task is blocked on the lock are
 * redirected to operate on task_struct::saved_state to ensure that these
 * are not dropped. On restore task_struct::saved_state is set to
 * TASK_RUNNING so any wakeup attempt redirected to saved_state will fail.
 *
 * The lock operation looks like this:
 *
 *        current_save_and_set_rtlock_wait_state();
 *        for (;;) {
 *                if (try_lock())
 *                        break;
 *                raw_spin_unlock_irq(&lock->wait_lock);
 *                schedule_rtlock();
 *                raw_spin_lock_irq(&lock->wait_lock);
 *                set_current_state(TASK_RTLOCK_WAIT);
 *        }
 *        current_restore_rtlock_saved_state();
 */
#define current_save_and_set_rtlock_wait_state()                        \
        do {                                                                \
                lockdep_assert_irqs_disabled();                                \
                raw_spin_lock(&current->pi_lock);                        \
                current->saved_state = current->__state;                \
                debug_rtlock_wait_set_state();                                \
                trace_set_current_state(TASK_RTLOCK_WAIT);                \
                WRITE_ONCE(current->__state, TASK_RTLOCK_WAIT);                \
                raw_spin_unlock(&current->pi_lock);                        \
        } while (0);

#define current_restore_rtlock_saved_state()                                \
        do {                                                                \
                lockdep_assert_irqs_disabled();                                \
                raw_spin_lock(&current->pi_lock);                        \
                debug_rtlock_wait_restore_state();                        \
                trace_set_current_state(current->saved_state);                \
                WRITE_ONCE(current->__state, current->saved_state);        \
                current->saved_state = TASK_RUNNING;                        \
                raw_spin_unlock(&current->pi_lock);                        \
        } while (0);

#define get_current_state()        READ_ONCE(current->__state)

/*
 * Define the task command name length as enum, then it can be visible to
 * BPF programs.
 */
enum {
        TASK_COMM_LEN = 16,
};

extern void sched_tick(void);

#define        MAX_SCHEDULE_TIMEOUT                LONG_MAX

extern long schedule_timeout(long timeout);
extern long schedule_timeout_interruptible(long timeout);
extern long schedule_timeout_killable(long timeout);
extern long schedule_timeout_uninterruptible(long timeout);
extern long schedule_timeout_idle(long timeout);
asmlinkage void schedule(void);
extern void schedule_preempt_disabled(void);
asmlinkage void preempt_schedule_irq(void);
#ifdef CONFIG_PREEMPT_RT
 extern void schedule_rtlock(void);
#endif

extern int __must_check io_schedule_prepare(void);
extern void io_schedule_finish(int token);
extern long io_schedule_timeout(long timeout);
extern void io_schedule(void);

/* wrapper function to trace from this header file */
DECLARE_TRACEPOINT(sched_set_state_tp);
extern void __trace_set_current_state(int state_value);

/**
 * struct prev_cputime - snapshot of system and user cputime
 * @utime: time spent in user mode
 * @stime: time spent in system mode
 * @lock: protects the above two fields
 *
 * Stores previous user/system time values such that we can guarantee
 * monotonicity.
 */
struct prev_cputime {
#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
        u64                                utime;
        u64                                stime;
        raw_spinlock_t                        lock;
#endif
};

enum vtime_state {
        /* Task is sleeping or running in a CPU with VTIME inactive: */
        VTIME_INACTIVE = 0,
        /* Task is idle */
        VTIME_IDLE,
        /* Task runs in kernelspace in a CPU with VTIME active: */
        VTIME_SYS,
        /* Task runs in userspace in a CPU with VTIME active: */
        VTIME_USER,
        /* Task runs as guests in a CPU with VTIME active: */
        VTIME_GUEST,
};

struct vtime {
        seqcount_t                seqcount;
        unsigned long long        starttime;
        enum vtime_state        state;
        unsigned int                cpu;
        u64                        utime;
        u64                        stime;
        u64                        gtime;
};

/*
 * Utilization clamp constraints.
 * @UCLAMP_MIN:        Minimum utilization
 * @UCLAMP_MAX:        Maximum utilization
 * @UCLAMP_CNT:        Utilization clamp constraints count
 */
enum uclamp_id {
        UCLAMP_MIN = 0,
        UCLAMP_MAX,
        UCLAMP_CNT
};

#ifdef CONFIG_SMP
extern struct root_domain def_root_domain;
extern struct mutex sched_domains_mutex;
extern void sched_domains_mutex_lock(void);
extern void sched_domains_mutex_unlock(void);
#else
static inline void sched_domains_mutex_lock(void) { }
static inline void sched_domains_mutex_unlock(void) { }
#endif

struct sched_param {
        int sched_priority;
};

struct sched_info {
#ifdef CONFIG_SCHED_INFO
        /* Cumulative counters: */

        /* # of times we have run on this CPU: */
        unsigned long                        pcount;

        /* Time spent waiting on a runqueue: */
        unsigned long long                run_delay;

        /* Max time spent waiting on a runqueue: */
        unsigned long long                max_run_delay;

        /* Min time spent waiting on a runqueue: */
        unsigned long long                min_run_delay;

        /* Timestamps: */

        /* When did we last run on a CPU? */
        unsigned long long                last_arrival;

        /* When were we last queued to run? */
        unsigned long long                last_queued;

#endif /* CONFIG_SCHED_INFO */
};

/*
 * Integer metrics need fixed point arithmetic, e.g., sched/fair
 * has a few: load, load_avg, util_avg, freq, and capacity.
 *
 * We define a basic fixed point arithmetic range, and then formalize
 * all these metrics based on that basic range.
 */
# define SCHED_FIXEDPOINT_SHIFT                10
# define SCHED_FIXEDPOINT_SCALE                (1L << SCHED_FIXEDPOINT_SHIFT)

/* Increase resolution of cpu_capacity calculations */
# define SCHED_CAPACITY_SHIFT                SCHED_FIXEDPOINT_SHIFT
# define SCHED_CAPACITY_SCALE                (1L << SCHED_CAPACITY_SHIFT)

struct load_weight {
        unsigned long                        weight;
        u32                                inv_weight;
};

/*
 * The load/runnable/util_avg accumulates an infinite geometric series
 * (see __update_load_avg_cfs_rq() in kernel/sched/pelt.c).
 *
 * [load_avg definition]
 *
 *   load_avg = runnable% * scale_load_down(load)
 *
 * [runnable_avg definition]
 *
 *   runnable_avg = runnable% * SCHED_CAPACITY_SCALE
 *
 * [util_avg definition]
 *
 *   util_avg = running% * SCHED_CAPACITY_SCALE
 *
 * where runnable% is the time ratio that a sched_entity is runnable and
 * running% the time ratio that a sched_entity is running.
 *
 * For cfs_rq, they are the aggregated values of all runnable and blocked
 * sched_entities.
 *
 * The load/runnable/util_avg doesn't directly factor frequency scaling and CPU
 * capacity scaling. The scaling is done through the rq_clock_pelt that is used
 * for computing those signals (see update_rq_clock_pelt())
 *
 * N.B., the above ratios (runnable% and running%) themselves are in the
 * range of [0, 1]. To do fixed point arithmetics, we therefore scale them
 * to as large a range as necessary. This is for example reflected by
 * util_avg's SCHED_CAPACITY_SCALE.
 *
 * [Overflow issue]
 *
 * The 64-bit load_sum can have 4353082796 (=2^64/47742/88761) entities
 * with the highest load (=88761), always runnable on a single cfs_rq,
 * and should not overflow as the number already hits PID_MAX_LIMIT.
 *
 * For all other cases (including 32-bit kernels), struct load_weight's
 * weight will overflow first before we do, because:
 *
 *    Max(load_avg) <= Max(load.weight)
 *
 * Then it is the load_weight's responsibility to consider overflow
 * issues.
 */
struct sched_avg {
        u64                                last_update_time;
        u64                                load_sum;
        u64                                runnable_sum;
        u32                                util_sum;
        u32                                period_contrib;
        unsigned long                        load_avg;
        unsigned long                        runnable_avg;
        unsigned long                        util_avg;
        unsigned int                        util_est;
} ____cacheline_aligned;

/*
 * The UTIL_AVG_UNCHANGED flag is used to synchronize util_est with util_avg
 * updates. When a task is dequeued, its util_est should not be updated if its
 * util_avg has not been updated in the meantime.
 * This information is mapped into the MSB bit of util_est at dequeue time.
 * Since max value of util_est for a task is 1024 (PELT util_avg for a task)
 * it is safe to use MSB.
 */
#define UTIL_EST_WEIGHT_SHIFT                2
#define UTIL_AVG_UNCHANGED                0x80000000

struct sched_statistics {
#ifdef CONFIG_SCHEDSTATS
        u64                                wait_start;
        u64                                wait_max;
        u64                                wait_count;
        u64                                wait_sum;
        u64                                iowait_count;
        u64                                iowait_sum;

        u64                                sleep_start;
        u64                                sleep_max;
        s64                                sum_sleep_runtime;

        u64                                block_start;
        u64                                block_max;
        s64                                sum_block_runtime;

        s64                                exec_max;
        u64                                slice_max;

        u64                                nr_migrations_cold;
        u64                                nr_failed_migrations_affine;
        u64                                nr_failed_migrations_running;
        u64                                nr_failed_migrations_hot;
        u64                                nr_forced_migrations;

        u64                                nr_wakeups;
        u64                                nr_wakeups_sync;
        u64                                nr_wakeups_migrate;
        u64                                nr_wakeups_local;
        u64                                nr_wakeups_remote;
        u64                                nr_wakeups_affine;
        u64                                nr_wakeups_affine_attempts;
        u64                                nr_wakeups_passive;
        u64                                nr_wakeups_idle;

#ifdef CONFIG_SCHED_CORE
        u64                                core_forceidle_sum;
#endif
#endif /* CONFIG_SCHEDSTATS */
} ____cacheline_aligned;

struct sched_entity {
        /* For load-balancing: */
        struct load_weight                load;
        struct rb_node                        run_node;
        u64                                deadline;
        u64                                min_vruntime;
        u64                                min_slice;

        struct list_head                group_node;
        unsigned char                        on_rq;
        unsigned char                        sched_delayed;
        unsigned char                        rel_deadline;
        unsigned char                        custom_slice;
                                        /* hole */

        u64                                exec_start;
        u64                                sum_exec_runtime;
        u64                                prev_sum_exec_runtime;
        u64                                vruntime;
        s64                                vlag;
        u64                                slice;

        u64                                nr_migrations;

#ifdef CONFIG_FAIR_GROUP_SCHED
        int                                depth;
        struct sched_entity                *parent;
        /* rq on which this entity is (to be) queued: */
        struct cfs_rq                        *cfs_rq;
        /* rq "owned" by this entity/group: */
        struct cfs_rq                        *my_q;
        /* cached value of my_q->h_nr_running */
        unsigned long                        runnable_weight;
#endif

#ifdef CONFIG_SMP
        /*
         * Per entity load average tracking.
         *
         * Put into separate cache line so it does not
         * collide with read-mostly values above.
         */
        struct sched_avg                avg;
#endif
};

struct sched_rt_entity {
        struct list_head                run_list;
        unsigned long                        timeout;
        unsigned long                        watchdog_stamp;
        unsigned int                        time_slice;
        unsigned short                        on_rq;
        unsigned short                        on_list;

        struct sched_rt_entity                *back;
#ifdef CONFIG_RT_GROUP_SCHED
        struct sched_rt_entity                *parent;
        /* rq on which this entity is (to be) queued: */
        struct rt_rq                        *rt_rq;
        /* rq "owned" by this entity/group: */
        struct rt_rq                        *my_q;
#endif
} __randomize_layout;

typedef bool (*dl_server_has_tasks_f)(struct sched_dl_entity *);
typedef struct task_struct *(*dl_server_pick_f)(struct sched_dl_entity *);

struct sched_dl_entity {
        struct rb_node                        rb_node;

        /*
         * Original scheduling parameters. Copied here from sched_attr
         * during sched_setattr(), they will remain the same until
         * the next sched_setattr().
         */
        u64                                dl_runtime;        /* Maximum runtime for each instance        */
        u64                                dl_deadline;        /* Relative deadline of each instance        */
        u64                                dl_period;        /* Separation of two instances (period) */
        u64                                dl_bw;                /* dl_runtime / dl_period                */
        u64                                dl_density;        /* dl_runtime / dl_deadline                */

        /*
         * Actual scheduling parameters. Initialized with the values above,
         * they are continuously updated during task execution. Note that
         * the remaining runtime could be < 0 in case we are in overrun.
         */
        s64                                runtime;        /* Remaining runtime for this instance        */
        u64                                deadline;        /* Absolute deadline for this instance        */
        unsigned int                        flags;                /* Specifying the scheduler behaviour        */

        /*
         * Some bool flags:
         *
         * @dl_throttled tells if we exhausted the runtime. If so, the
         * task has to wait for a replenishment to be performed at the
         * next firing of dl_timer.
         *
         * @dl_yielded tells if task gave up the CPU before consuming
         * all its available runtime during the last job.
         *
         * @dl_non_contending tells if the task is inactive while still
         * contributing to the active utilization. In other words, it
         * indicates if the inactive timer has been armed and its handler
         * has not been executed yet. This flag is useful to avoid race
         * conditions between the inactive timer handler and the wakeup
         * code.
         *
         * @dl_overrun tells if the task asked to be informed about runtime
         * overruns.
         *
         * @dl_server tells if this is a server entity.
         *
         * @dl_defer tells if this is a deferred or regular server. For
         * now only defer server exists.
         *
         * @dl_defer_armed tells if the deferrable server is waiting
         * for the replenishment timer to activate it.
         *
         * @dl_server_active tells if the dlserver is active(started).
         * dlserver is started on first cfs enqueue on an idle runqueue
         * and is stopped when a dequeue results in 0 cfs tasks on the
         * runqueue. In other words, dlserver is active only when cpu's
         * runqueue has atleast one cfs task.
         *
         * @dl_defer_running tells if the deferrable server is actually
         * running, skipping the defer phase.
         */
        unsigned int                        dl_throttled      : 1;
        unsigned int                        dl_yielded        : 1;
        unsigned int                        dl_non_contending : 1;
        unsigned int                        dl_overrun          : 1;
        unsigned int                        dl_server         : 1;
        unsigned int                        dl_server_active  : 1;
        unsigned int                        dl_defer          : 1;
        unsigned int                        dl_defer_armed          : 1;
        unsigned int                        dl_defer_running  : 1;

        /*
         * Bandwidth enforcement timer. Each -deadline task has its
         * own bandwidth to be enforced, thus we need one timer per task.
         */
        struct hrtimer                        dl_timer;

        /*
         * Inactive timer, responsible for decreasing the active utilization
         * at the "0-lag time". When a -deadline task blocks, it contributes
         * to GRUB's active utilization until the "0-lag time", hence a
         * timer is needed to decrease the active utilization at the correct
         * time.
         */
        struct hrtimer                        inactive_timer;

        /*
         * Bits for DL-server functionality. Also see the comment near
         * dl_server_update().
         *
         * @rq the runqueue this server is for
         *
         * @server_has_tasks() returns true if @server_pick return a
         * runnable task.
         */
        struct rq                        *rq;
        dl_server_has_tasks_f                server_has_tasks;
        dl_server_pick_f                server_pick_task;

#ifdef CONFIG_RT_MUTEXES
        /*
         * Priority Inheritance. When a DEADLINE scheduling entity is boosted
         * pi_se points to the donor, otherwise points to the dl_se it belongs
         * to (the original one/itself).
         */
        struct sched_dl_entity *pi_se;
#endif
};

#ifdef CONFIG_UCLAMP_TASK
/* Number of utilization clamp buckets (shorter alias) */
#define UCLAMP_BUCKETS CONFIG_UCLAMP_BUCKETS_COUNT

/*
 * Utilization clamp for a scheduling entity
 * @value:                clamp value "assigned" to a se
 * @bucket_id:                bucket index corresponding to the "assigned" value
 * @active:                the se is currently refcounted in a rq's bucket
 * @user_defined:        the requested clamp value comes from user-space
 *
 * The bucket_id is the index of the clamp bucket matching the clamp value
 * which is pre-computed and stored to avoid expensive integer divisions from
 * the fast path.
 *
 * The active bit is set whenever a task has got an "effective" value assigned,
 * which can be different from the clamp value "requested" from user-space.
 * This allows to know a task is refcounted in the rq's bucket corresponding
 * to the "effective" bucket_id.
 *
 * The user_defined bit is set whenever a task has got a task-specific clamp
 * value requested from userspace, i.e. the system defaults apply to this task
 * just as a restriction. This allows to relax default clamps when a less
 * restrictive task-specific value has been requested, thus allowing to
 * implement a "nice" semantic. For example, a task running with a 20%
 * default boost can still drop its own boosting to 0%.
 */
struct uclamp_se {
        unsigned int value                : bits_per(SCHED_CAPACITY_SCALE);
        unsigned int bucket_id                : bits_per(UCLAMP_BUCKETS);
        unsigned int active                : 1;
        unsigned int user_defined        : 1;
};
#endif /* CONFIG_UCLAMP_TASK */

union rcu_special {
        struct {
                u8                        blocked;
                u8                        need_qs;
                u8                        exp_hint; /* Hint for performance. */
                u8                        need_mb; /* Readers need smp_mb(). */
        } b; /* Bits. */
        u32 s; /* Set of bits. */
};

enum perf_event_task_context {
        perf_invalid_context = -1,
        perf_hw_context = 0,
        perf_sw_context,
        perf_nr_task_contexts,
};

/*
 * Number of contexts where an event can trigger:
 *      task, softirq, hardirq, nmi.
 */
#define PERF_NR_CONTEXTS        4

struct wake_q_node {
        struct wake_q_node *next;
};

struct kmap_ctrl {
#ifdef CONFIG_KMAP_LOCAL
        int                                idx;
        pte_t                                pteval[KM_MAX_IDX];
#endif
};

struct task_struct {
#ifdef CONFIG_THREAD_INFO_IN_TASK
        /*
         * For reasons of header soup (see current_thread_info()), this
         * must be the first element of task_struct.
         */
        struct thread_info                thread_info;
#endif
        unsigned int                        __state;

        /* saved state for "spinlock sleepers" */
        unsigned int                        saved_state;

        /*
         * This begins the randomizable portion of task_struct. Only
         * scheduling-critical items should be added above here.
         */
        randomized_struct_fields_start

        void                                *stack;
        refcount_t                        usage;
        /* Per task flags (PF_*), defined further below: */
        unsigned int                        flags;
        unsigned int                        ptrace;

#ifdef CONFIG_MEM_ALLOC_PROFILING
        struct alloc_tag                *alloc_tag;
#endif

#ifdef CONFIG_SMP
        int                                on_cpu;
        struct __call_single_node        wake_entry;
        unsigned int                        wakee_flips;
        unsigned long                        wakee_flip_decay_ts;
        struct task_struct                *last_wakee;

        /*
         * recent_used_cpu is initially set as the last CPU used by a task
         * that wakes affine another task. Waker/wakee relationships can
         * push tasks around a CPU where each wakeup moves to the next one.
         * Tracking a recently used CPU allows a quick search for a recently
         * used CPU that may be idle.
         */
        int                                recent_used_cpu;
        int                                wake_cpu;
#endif
        int                                on_rq;

        int                                prio;
        int                                static_prio;
        int                                normal_prio;
        unsigned int                        rt_priority;

        struct sched_entity                se;
        struct sched_rt_entity                rt;
        struct sched_dl_entity                dl;
        struct sched_dl_entity                *dl_server;
#ifdef CONFIG_SCHED_CLASS_EXT
        struct sched_ext_entity                scx;
#endif
        const struct sched_class        *sched_class;

#ifdef CONFIG_SCHED_CORE
        struct rb_node                        core_node;
        unsigned long                        core_cookie;
        unsigned int                        core_occupation;
#endif

#ifdef CONFIG_CGROUP_SCHED
        struct task_group                *sched_task_group;
#endif


#ifdef CONFIG_UCLAMP_TASK
        /*
         * Clamp values requested for a scheduling entity.
         * Must be updated with task_rq_lock() held.
         */
        struct uclamp_se                uclamp_req[UCLAMP_CNT];
        /*
         * Effective clamp values used for a scheduling entity.
         * Must be updated with task_rq_lock() held.
         */
        struct uclamp_se                uclamp[UCLAMP_CNT];
#endif

        struct sched_statistics         stats;

#ifdef CONFIG_PREEMPT_NOTIFIERS
        /* List of struct preempt_notifier: */
        struct hlist_head                preempt_notifiers;
#endif

#ifdef CONFIG_BLK_DEV_IO_TRACE
        unsigned int                        btrace_seq;
#endif

        unsigned int                        policy;
        unsigned long                        max_allowed_capacity;
        int                                nr_cpus_allowed;
        const cpumask_t                        *cpus_ptr;
        cpumask_t                        *user_cpus_ptr;
        cpumask_t                        cpus_mask;
        void                                *migration_pending;
#ifdef CONFIG_SMP
        unsigned short                        migration_disabled;
#endif
        unsigned short                        migration_flags;

#ifdef CONFIG_PREEMPT_RCU
        int                                rcu_read_lock_nesting;
        union rcu_special                rcu_read_unlock_special;
        struct list_head                rcu_node_entry;
        struct rcu_node                        *rcu_blocked_node;
#endif /* #ifdef CONFIG_PREEMPT_RCU */

#ifdef CONFIG_TASKS_RCU
        unsigned long                        rcu_tasks_nvcsw;
        u8                                rcu_tasks_holdout;
        u8                                rcu_tasks_idx;
        int                                rcu_tasks_idle_cpu;
        struct list_head                rcu_tasks_holdout_list;
        int                                rcu_tasks_exit_cpu;
        struct list_head                rcu_tasks_exit_list;
#endif /* #ifdef CONFIG_TASKS_RCU */

#ifdef CONFIG_TASKS_TRACE_RCU
        int                                trc_reader_nesting;
        int                                trc_ipi_to_cpu;
        union rcu_special                trc_reader_special;
        struct list_head                trc_holdout_list;
        struct list_head                trc_blkd_node;
        int                                trc_blkd_cpu;
#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */

        struct sched_info                sched_info;

        struct list_head                tasks;
#ifdef CONFIG_SMP
        struct plist_node                pushable_tasks;
        struct rb_node                        pushable_dl_tasks;
#endif

        struct mm_struct                *mm;
        struct mm_struct                *active_mm;
        struct address_space                *faults_disabled_mapping;

        int                                exit_state;
        int                                exit_code;
        int                                exit_signal;
        /* The signal sent when the parent dies: */
        int                                pdeath_signal;
        /* JOBCTL_*, siglock protected: */
        unsigned long                        jobctl;

        /* Used for emulating ABI behavior of previous Linux versions: */
        unsigned int                        personality;

        /* Scheduler bits, serialized by scheduler locks: */
        unsigned                        sched_reset_on_fork:1;
        unsigned                        sched_contributes_to_load:1;
        unsigned                        sched_migrated:1;
        unsigned                        sched_task_hot:1;

        /* Force alignment to the next boundary: */
        unsigned                        :0;

        /* Unserialized, strictly 'current' */

        /*
         * This field must not be in the scheduler word above due to wakelist
         * queueing no longer being serialized by p->on_cpu. However:
         *
         * p->XXX = X;                        ttwu()
         * schedule()                          if (p->on_rq && ..) // false
         *   smp_mb__after_spinlock();          if (smp_load_acquire(&p->on_cpu) && //true
         *   deactivate_task()                      ttwu_queue_wakelist())
         *     p->on_rq = 0;                        p->sched_remote_wakeup = Y;
         *
         * guarantees all stores of 'current' are visible before
         * ->sched_remote_wakeup gets used, so it can be in this word.
         */
        unsigned                        sched_remote_wakeup:1;
#ifdef CONFIG_RT_MUTEXES
        unsigned                        sched_rt_mutex:1;
#endif

        /* Bit to tell TOMOYO we're in execve(): */
        unsigned                        in_execve:1;
        unsigned                        in_iowait:1;
#ifndef TIF_RESTORE_SIGMASK
        unsigned                        restore_sigmask:1;
#endif
#ifdef CONFIG_MEMCG_V1
        unsigned                        in_user_fault:1;
#endif
#ifdef CONFIG_LRU_GEN
        /* whether the LRU algorithm may apply to this access */
        unsigned                        in_lru_fault:1;
#endif
#ifdef CONFIG_COMPAT_BRK
        unsigned                        brk_randomized:1;
#endif
#ifdef CONFIG_CGROUPS
        /* disallow userland-initiated cgroup migration */
        unsigned                        no_cgroup_migration:1;
        /* task is frozen/stopped (used by the cgroup freezer) */
        unsigned                        frozen:1;
#endif
#ifdef CONFIG_BLK_CGROUP
        unsigned                        use_memdelay:1;
#endif
#ifdef CONFIG_PSI
        /* Stalled due to lack of memory */
        unsigned                        in_memstall:1;
#endif
#ifdef CONFIG_PAGE_OWNER
        /* Used by page_owner=on to detect recursion in page tracking. */
        unsigned                        in_page_owner:1;
#endif
#ifdef CONFIG_EVENTFD
        /* Recursion prevention for eventfd_signal() */
        unsigned                        in_eventfd:1;
#endif
#ifdef CONFIG_ARCH_HAS_CPU_PASID
        unsigned                        pasid_activated:1;
#endif
#ifdef CONFIG_X86_BUS_LOCK_DETECT
        unsigned                        reported_split_lock:1;
#endif
#ifdef CONFIG_TASK_DELAY_ACCT
        /* delay due to memory thrashing */
        unsigned                        in_thrashing:1;
#endif
#ifdef CONFIG_PREEMPT_RT
        struct netdev_xmit                net_xmit;
#endif
        unsigned long                        atomic_flags; /* Flags requiring atomic access. */

        struct restart_block                restart_block;

        pid_t                                pid;
        pid_t                                tgid;

#ifdef CONFIG_STACKPROTECTOR
        /* Canary value for the -fstack-protector GCC feature: */
        unsigned long                        stack_canary;
#endif
        /*
         * Pointers to the (original) parent process, youngest child, younger sibling,
         * older sibling, respectively.  (p->father can be replaced with
         * p->real_parent->pid)
         */

        /* Real parent process: */
        struct task_struct __rcu        *real_parent;

        /* Recipient of SIGCHLD, wait4() reports: */
        struct task_struct __rcu        *parent;

        /*
         * Children/sibling form the list of natural children:
         */
        struct list_head                children;
        struct list_head                sibling;
        struct task_struct                *group_leader;

        /*
         * 'ptraced' is the list of tasks this task is using ptrace() on.
         *
         * This includes both natural children and PTRACE_ATTACH targets.
         * 'ptrace_entry' is this task's link on the p->parent->ptraced list.
         */
        struct list_head                ptraced;
        struct list_head                ptrace_entry;

        /* PID/PID hash table linkage. */
        struct pid                        *thread_pid;
        struct hlist_node                pid_links[PIDTYPE_MAX];
        struct list_head                thread_node;

        struct completion                *vfork_done;

        /* CLONE_CHILD_SETTID: */
        int __user                        *set_child_tid;

        /* CLONE_CHILD_CLEARTID: */
        int __user                        *clear_child_tid;

        /* PF_KTHREAD | PF_IO_WORKER */
        void                                *worker_private;

        u64                                utime;
        u64                                stime;
#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
        u64                                utimescaled;
        u64                                stimescaled;
#endif
        u64                                gtime;
        struct prev_cputime                prev_cputime;
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
        struct vtime                        vtime;
#endif

#ifdef CONFIG_NO_HZ_FULL
        atomic_t                        tick_dep_mask;
#endif
        /* Context switch counts: */
        unsigned long                        nvcsw;
        unsigned long                        nivcsw;

        /* Monotonic time in nsecs: */
        u64                                start_time;

        /* Boot based time in nsecs: */
        u64                                start_boottime;

        /* MM fault and swap info: this can arguably be seen as either mm-specific or thread-specific: */
        unsigned long                        min_flt;
        unsigned long                        maj_flt;

        /* Empty if CONFIG_POSIX_CPUTIMERS=n */
        struct posix_cputimers                posix_cputimers;

#ifdef CONFIG_POSIX_CPU_TIMERS_TASK_WORK
        struct posix_cputimers_work        posix_cputimers_work;
#endif

        /* Process credentials: */

        /* Tracer's credentials at attach: */
        const struct cred __rcu                *ptracer_cred;

        /* Objective and real subjective task credentials (COW): */
        const struct cred __rcu                *real_cred;

        /* Effective (overridable) subjective task credentials (COW): */
        const struct cred __rcu                *cred;

#ifdef CONFIG_KEYS
        /* Cached requested key. */
        struct key                        *cached_requested_key;
#endif

        /*
         * executable name, excluding path.
         *
         * - normally initialized begin_new_exec()
         * - set it with set_task_comm()
         *   - strscpy_pad() to ensure it is always NUL-terminated and
         *     zero-padded
         *   - task_lock() to ensure the operation is atomic and the name is
         *     fully updated.
         */
        char                                comm[TASK_COMM_LEN];

        struct nameidata                *nameidata;

#ifdef CONFIG_SYSVIPC
        struct sysv_sem                        sysvsem;
        struct sysv_shm                        sysvshm;
#endif
#ifdef CONFIG_DETECT_HUNG_TASK
        unsigned long                        last_switch_count;
        unsigned long                        last_switch_time;
#endif
        /* Filesystem information: */
        struct fs_struct                *fs;

        /* Open file information: */
        struct files_struct                *files;

#ifdef CONFIG_IO_URING
        struct io_uring_task                *io_uring;
#endif

        /* Namespaces: */
        struct nsproxy                        *nsproxy;

        /* Signal handlers: */
        struct signal_struct                *signal;
        struct sighand_struct __rcu                *sighand;
        sigset_t                        blocked;
        sigset_t                        real_blocked;
        /* Restored if set_restore_sigmask() was used: */
        sigset_t                        saved_sigmask;
        struct sigpending                pending;
        unsigned long                        sas_ss_sp;
        size_t                                sas_ss_size;
        unsigned int                        sas_ss_flags;

        struct callback_head                *task_works;

#ifdef CONFIG_AUDIT
#ifdef CONFIG_AUDITSYSCALL
        struct audit_context                *audit_context;
#endif
        kuid_t                                loginuid;
        unsigned int                        sessionid;
#endif
        struct seccomp                        seccomp;
        struct syscall_user_dispatch        syscall_dispatch;

        /* Thread group tracking: */
        u64                                parent_exec_id;
        u64                                self_exec_id;

        /* Protection against (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed, mempolicy: */
        spinlock_t                        alloc_lock;

        /* Protection of the PI data structures: */
        raw_spinlock_t                        pi_lock;

        struct wake_q_node                wake_q;

#ifdef CONFIG_RT_MUTEXES
        /* PI waiters blocked on a rt_mutex held by this task: */
        struct rb_root_cached                pi_waiters;
        /* Updated under owner's pi_lock and rq lock */
        struct task_struct                *pi_top_task;
        /* Deadlock detection and priority inheritance handling: */
        struct rt_mutex_waiter                *pi_blocked_on;
#endif

#ifdef CONFIG_DEBUG_MUTEXES
        /* Mutex deadlock detection: */
        struct mutex_waiter                *blocked_on;
#endif

#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
        struct mutex                        *blocker_mutex;
#endif

#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
        int                                non_block_count;
#endif

#ifdef CONFIG_TRACE_IRQFLAGS
        struct irqtrace_events                irqtrace;
        unsigned int                        hardirq_threaded;
        u64                                hardirq_chain_key;
        int                                softirqs_enabled;
        int                                softirq_context;
        int                                irq_config;
#endif
#ifdef CONFIG_PREEMPT_RT
        int                                softirq_disable_cnt;
#endif

#ifdef CONFIG_LOCKDEP
# define MAX_LOCK_DEPTH                        48UL
        u64                                curr_chain_key;
        int                                lockdep_depth;
        unsigned int                        lockdep_recursion;
        struct held_lock                held_locks[MAX_LOCK_DEPTH];
#endif

#if defined(CONFIG_UBSAN) && !defined(CONFIG_UBSAN_TRAP)
        unsigned int                        in_ubsan;
#endif

        /* Journalling filesystem info: */
        void                                *journal_info;

        /* Stacked block device info: */
        struct bio_list                        *bio_list;

        /* Stack plugging: */
        struct blk_plug                        *plug;

        /* VM state: */
        struct reclaim_state                *reclaim_state;

        struct io_context                *io_context;

#ifdef CONFIG_COMPACTION
        struct capture_control                *capture_control;
#endif
        /* Ptrace state: */
        unsigned long                        ptrace_message;
        kernel_siginfo_t                *last_siginfo;

        struct task_io_accounting        ioac;
#ifdef CONFIG_PSI
        /* Pressure stall state */
        unsigned int                        psi_flags;
#endif
#ifdef CONFIG_TASK_XACCT
        /* Accumulated RSS usage: */
        u64                                acct_rss_mem1;
        /* Accumulated virtual memory usage: */
        u64                                acct_vm_mem1;
        /* stime + utime since last update: */
        u64                                acct_timexpd;
#endif
#ifdef CONFIG_CPUSETS
        /* Protected by ->alloc_lock: */
        nodemask_t                        mems_allowed;
        /* Sequence number to catch updates: */
        seqcount_spinlock_t                mems_allowed_seq;
        int                                cpuset_mem_spread_rotor;
#endif
#ifdef CONFIG_CGROUPS
        /* Control Group info protected by css_set_lock: */
        struct css_set __rcu                *cgroups;
        /* cg_list protected by css_set_lock and tsk->alloc_lock: */
        struct list_head                cg_list;
#endif
#ifdef CONFIG_X86_CPU_RESCTRL
        u32                                closid;
        u32                                rmid;
#endif
#ifdef CONFIG_FUTEX
        struct robust_list_head __user        *robust_list;
#ifdef CONFIG_COMPAT
        struct compat_robust_list_head __user *compat_robust_list;
#endif
        struct list_head                pi_state_list;
        struct futex_pi_state                *pi_state_cache;
        struct mutex                        futex_exit_mutex;
        unsigned int                        futex_state;
#endif
#ifdef CONFIG_PERF_EVENTS
        u8                                perf_recursion[PERF_NR_CONTEXTS];
        struct perf_event_context        *perf_event_ctxp;
        struct mutex                        perf_event_mutex;
        struct list_head                perf_event_list;
        struct perf_ctx_data __rcu        *perf_ctx_data;
#endif
#ifdef CONFIG_DEBUG_PREEMPT
        unsigned long                        preempt_disable_ip;
#endif
#ifdef CONFIG_NUMA
        /* Protected by alloc_lock: */
        struct mempolicy                *mempolicy;
        short                                il_prev;
        u8                                il_weight;
        short                                pref_node_fork;
#endif
#ifdef CONFIG_NUMA_BALANCING
        int                                numa_scan_seq;
        unsigned int                        numa_scan_period;
        unsigned int                        numa_scan_period_max;
        int                                numa_preferred_nid;
        unsigned long                        numa_migrate_retry;
        /* Migration stamp: */
        u64                                node_stamp;
        u64                                last_task_numa_placement;
        u64                                last_sum_exec_runtime;
        struct callback_head                numa_work;

        /*
         * This pointer is only modified for current in syscall and
         * pagefault context (and for tasks being destroyed), so it can be read
         * from any of the following contexts:
         *  - RCU read-side critical section
         *  - current->numa_group from everywhere
         *  - task's runqueue locked, task not running
         */
        struct numa_group __rcu                *numa_group;

        /*
         * numa_faults is an array split into four regions:
         * faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer
         * in this precise order.
         *
         * faults_memory: Exponential decaying average of faults on a per-node
         * basis. Scheduling placement decisions are made based on these
         * counts. The values remain static for the duration of a PTE scan.
         * faults_cpu: Track the nodes the process was running on when a NUMA
         * hinting fault was incurred.
         * faults_memory_buffer and faults_cpu_buffer: Record faults per node
         * during the current scan window. When the scan completes, the counts
         * in faults_memory and faults_cpu decay and these values are copied.
         */
        unsigned long                        *numa_faults;
        unsigned long                        total_numa_faults;

        /*
         * numa_faults_locality tracks if faults recorded during the last
         * scan window were remote/local or failed to migrate. The task scan
         * period is adapted based on the locality of the faults with different
         * weights depending on whether they were shared or private faults
         */
        unsigned long                        numa_faults_locality[3];

        unsigned long                        numa_pages_migrated;
#endif /* CONFIG_NUMA_BALANCING */

#ifdef CONFIG_RSEQ
        struct rseq __user *rseq;
        u32 rseq_len;
        u32 rseq_sig;
        /*
         * RmW on rseq_event_mask must be performed atomically
         * with respect to preemption.
         */
        unsigned long rseq_event_mask;
# ifdef CONFIG_DEBUG_RSEQ
        /*
         * This is a place holder to save a copy of the rseq fields for
         * validation of read-only fields. The struct rseq has a
         * variable-length array at the end, so it cannot be used
         * directly. Reserve a size large enough for the known fields.
         */
        char                                rseq_fields[sizeof(struct rseq)];
# endif
#endif

#ifdef CONFIG_SCHED_MM_CID
        int                                mm_cid;                /* Current cid in mm */
        int                                last_mm_cid;        /* Most recent cid in mm */
        int                                migrate_from_cpu;
        int                                mm_cid_active;        /* Whether cid bitmap is active */
        struct callback_head                cid_work;
#endif

        struct tlbflush_unmap_batch        tlb_ubc;

        /* Cache last used pipe for splice(): */
        struct pipe_inode_info                *splice_pipe;

        struct page_frag                task_frag;

#ifdef CONFIG_TASK_DELAY_ACCT
        struct task_delay_info                *delays;
#endif

#ifdef CONFIG_FAULT_INJECTION
        int                                make_it_fail;
        unsigned int                        fail_nth;
#endif
        /*
         * When (nr_dirtied >= nr_dirtied_pause), it's time to call
         * balance_dirty_pages() for a dirty throttling pause:
         */
        int                                nr_dirtied;
        int                                nr_dirtied_pause;
        /* Start of a write-and-pause period: */
        unsigned long                        dirty_paused_when;

#ifdef CONFIG_LATENCYTOP
        int                                latency_record_count;
        struct latency_record                latency_record[LT_SAVECOUNT];
#endif
        /*
         * Time slack values; these are used to round up poll() and
         * select() etc timeout values. These are in nanoseconds.
         */
        u64                                timer_slack_ns;
        u64                                default_timer_slack_ns;

#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
        unsigned int                        kasan_depth;
#endif

#ifdef CONFIG_KCSAN
        struct kcsan_ctx                kcsan_ctx;
#ifdef CONFIG_TRACE_IRQFLAGS
        struct irqtrace_events                kcsan_save_irqtrace;
#endif
#ifdef CONFIG_KCSAN_WEAK_MEMORY
        int                                kcsan_stack_depth;
#endif
#endif

#ifdef CONFIG_KMSAN
        struct kmsan_ctx                kmsan_ctx;
#endif

#if IS_ENABLED(CONFIG_KUNIT)
        struct kunit                        *kunit_test;
#endif

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
        /* Index of current stored address in ret_stack: */
        int                                curr_ret_stack;
        int                                curr_ret_depth;

        /* Stack of return addresses for return function tracing: */
        unsigned long                        *ret_stack;

        /* Timestamp for last schedule: */
        unsigned long long                ftrace_timestamp;
        unsigned long long                ftrace_sleeptime;

        /*
         * Number of functions that haven't been traced
         * because of depth overrun:
         */
        atomic_t                        trace_overrun;

        /* Pause tracing: */
        atomic_t                        tracing_graph_pause;
#endif

#ifdef CONFIG_TRACING
        /* Bitmask and counter of trace recursion: */
        unsigned long                        trace_recursion;
#endif /* CONFIG_TRACING */

#ifdef CONFIG_KCOV
        /* See kernel/kcov.c for more details. */

        /* Coverage collection mode enabled for this task (0 if disabled): */
        unsigned int                        kcov_mode;

        /* Size of the kcov_area: */
        unsigned int                        kcov_size;

        /* Buffer for coverage collection: */
        void                                *kcov_area;

        /* KCOV descriptor wired with this task or NULL: */
        struct kcov                        *kcov;

        /* KCOV common handle for remote coverage collection: */
        u64                                kcov_handle;

        /* KCOV sequence number: */
        int                                kcov_sequence;

        /* Collect coverage from softirq context: */
        unsigned int                        kcov_softirq;
#endif

#ifdef CONFIG_MEMCG_V1
        struct mem_cgroup                *memcg_in_oom;
#endif

#ifdef CONFIG_MEMCG
        /* Number of pages to reclaim on returning to userland: */
        unsigned int                        memcg_nr_pages_over_high;

        /* Used by memcontrol for targeted memcg charge: */
        struct mem_cgroup                *active_memcg;

        /* Cache for current->cgroups->memcg->objcg lookups: */
        struct obj_cgroup                *objcg;
#endif

#ifdef CONFIG_BLK_CGROUP
        struct gendisk                        *throttle_disk;
#endif

#ifdef CONFIG_UPROBES
        struct uprobe_task                *utask;
#endif
#if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE)
        unsigned int                        sequential_io;
        unsigned int                        sequential_io_avg;
#endif
        struct kmap_ctrl                kmap_ctrl;
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
        unsigned long                        task_state_change;
# ifdef CONFIG_PREEMPT_RT
        unsigned long                        saved_state_change;
# endif
#endif
        struct rcu_head                        rcu;
        refcount_t                        rcu_users;
        int                                pagefault_disabled;
#ifdef CONFIG_MMU
        struct task_struct                *oom_reaper_list;
        struct timer_list                oom_reaper_timer;
#endif
#ifdef CONFIG_VMAP_STACK
        struct vm_struct                *stack_vm_area;
#endif
#ifdef CONFIG_THREAD_INFO_IN_TASK
        /* A live task holds one reference: */
        refcount_t                        stack_refcount;
#endif
#ifdef CONFIG_LIVEPATCH
        int patch_state;
#endif
#ifdef CONFIG_SECURITY
        /* Used by LSM modules for access restriction: */
        void                                *security;
#endif
#ifdef CONFIG_BPF_SYSCALL
        /* Used by BPF task local storage */
        struct bpf_local_storage __rcu        *bpf_storage;
        /* Used for BPF run context */
        struct bpf_run_ctx                *bpf_ctx;
#endif
        /* Used by BPF for per-TASK xdp storage */
        struct bpf_net_context                *bpf_net_context;

#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
        unsigned long                        lowest_stack;
        unsigned long                        prev_lowest_stack;
#endif

#ifdef CONFIG_X86_MCE
        void __user                        *mce_vaddr;
        __u64                                mce_kflags;
        u64                                mce_addr;
        __u64                                mce_ripv : 1,
                                        mce_whole_page : 1,
                                        __mce_reserved : 62;
        struct callback_head                mce_kill_me;
        int                                mce_count;
#endif

#ifdef CONFIG_KRETPROBES
        struct llist_head               kretprobe_instances;
#endif
#ifdef CONFIG_RETHOOK
        struct llist_head               rethooks;
#endif

#ifdef CONFIG_ARCH_HAS_PARANOID_L1D_FLUSH
        /*
         * If L1D flush is supported on mm context switch
         * then we use this callback head to queue kill work
         * to kill tasks that are not running on SMT disabled
         * cores
         */
        struct callback_head                l1d_flush_kill;
#endif

#ifdef CONFIG_RV
        /*
         * Per-task RV monitor. Nowadays fixed in RV_PER_TASK_MONITORS.
         * If we find justification for more monitors, we can think
         * about adding more or developing a dynamic method. So far,
         * none of these are justified.
         */
        union rv_task_monitor                rv[RV_PER_TASK_MONITORS];
#endif

#ifdef CONFIG_USER_EVENTS
        struct user_event_mm                *user_event_mm;
#endif

        /*
         * New fields for task_struct should be added above here, so that
         * they are included in the randomized portion of task_struct.
         */
        randomized_struct_fields_end

        /* CPU-specific state of this task: */
        struct thread_struct                thread;

        /*
         * WARNING: on x86, 'thread_struct' contains a variable-sized
         * structure.  It *MUST* be at the end of 'task_struct'.
         *
         * Do not put anything below here!
         */
};

#define TASK_REPORT_IDLE        (TASK_REPORT + 1)
#define TASK_REPORT_MAX                (TASK_REPORT_IDLE << 1)

static inline unsigned int __task_state_index(unsigned int tsk_state,
                                              unsigned int tsk_exit_state)
{
        unsigned int state = (tsk_state | tsk_exit_state) & TASK_REPORT;

        BUILD_BUG_ON_NOT_POWER_OF_2(TASK_REPORT_MAX);

        if ((tsk_state & TASK_IDLE) == TASK_IDLE)
                state = TASK_REPORT_IDLE;

        /*
         * We're lying here, but rather than expose a completely new task state
         * to userspace, we can make this appear as if the task has gone through
         * a regular rt_mutex_lock() call.
         * Report frozen tasks as uninterruptible.
         */
        if ((tsk_state & TASK_RTLOCK_WAIT) || (tsk_state & TASK_FROZEN))
                state = TASK_UNINTERRUPTIBLE;

        return fls(state);
}

static inline unsigned int task_state_index(struct task_struct *tsk)
{
        return __task_state_index(READ_ONCE(tsk->__state), tsk->exit_state);
}

static inline char task_index_to_char(unsigned int state)
{
        static const char state_char[] = "RSDTtXZPI";

        BUILD_BUG_ON(TASK_REPORT_MAX * 2 != 1 << (sizeof(state_char) - 1));

        return state_char[state];
}

static inline char task_state_to_char(struct task_struct *tsk)
{
        return task_index_to_char(task_state_index(tsk));
}

extern struct pid *cad_pid;

/*
 * Per process flags
 */
#define PF_VCPU                        0x00000001        /* I'm a virtual CPU */
#define PF_IDLE                        0x00000002        /* I am an IDLE thread */
#define PF_EXITING                0x00000004        /* Getting shut down */
#define PF_POSTCOREDUMP                0x00000008        /* Coredumps should ignore this task */
#define PF_IO_WORKER                0x00000010        /* Task is an IO worker */
#define PF_WQ_WORKER                0x00000020        /* I'm a workqueue worker */
#define PF_FORKNOEXEC                0x00000040        /* Forked but didn't exec */
#define PF_MCE_PROCESS                0x00000080      /* Process policy on mce errors */
#define PF_SUPERPRIV                0x00000100        /* Used super-user privileges */
#define PF_DUMPCORE                0x00000200        /* Dumped core */
#define PF_SIGNALED                0x00000400        /* Killed by a signal */
#define PF_MEMALLOC                0x00000800        /* Allocating memory to free memory. See memalloc_noreclaim_save() */
#define PF_NPROC_EXCEEDED        0x00001000        /* set_user() noticed that RLIMIT_NPROC was exceeded */
#define PF_USED_MATH                0x00002000        /* If unset the fpu must be initialized before use */
#define PF_USER_WORKER                0x00004000        /* Kernel thread cloned from userspace thread */
#define PF_NOFREEZE                0x00008000        /* This thread should not be frozen */
#define PF_KCOMPACTD                0x00010000        /* I am kcompactd */
#define PF_KSWAPD                0x00020000        /* I am kswapd */
#define PF_MEMALLOC_NOFS        0x00040000        /* All allocations inherit GFP_NOFS. See memalloc_nfs_save() */
#define PF_MEMALLOC_NOIO        0x00080000        /* All allocations inherit GFP_NOIO. See memalloc_noio_save() */
#define PF_LOCAL_THROTTLE        0x00100000        /* Throttle writes only against the bdi I write to,
                                                 * I am cleaning dirty pages from some other bdi. */
#define PF_KTHREAD                0x00200000        /* I am a kernel thread */
#define PF_RANDOMIZE                0x00400000        /* Randomize virtual address space */
#define PF__HOLE__00800000        0x00800000
#define PF__HOLE__01000000        0x01000000
#define PF__HOLE__02000000        0x02000000
#define PF_NO_SETAFFINITY        0x04000000        /* Userland is not allowed to meddle with cpus_mask */
#define PF_MCE_EARLY                0x08000000      /* Early kill for mce process policy */
#define PF_MEMALLOC_PIN                0x10000000        /* Allocations constrained to zones which allow long term pinning.
                                                 * See memalloc_pin_save() */
#define PF_BLOCK_TS                0x20000000        /* plug has ts that needs updating */
#define PF__HOLE__40000000        0x40000000
#define PF_SUSPEND_TASK                0x80000000      /* This thread called freeze_processes() and should not be frozen */

/*
 * Only the _current_ task can read/write to tsk->flags, but other
 * tasks can access tsk->flags in readonly mode for example
 * with tsk_used_math (like during threaded core dumping).
 * There is however an exception to this rule during ptrace
 * or during fork: the ptracer task is allowed to write to the
 * child->flags of its traced child (same goes for fork, the parent
 * can write to the child->flags), because we're guaranteed the
 * child is not running and in turn not changing child->flags
 * at the same time the parent does it.
 */
#define clear_stopped_child_used_math(child)        do { (child)->flags &= ~PF_USED_MATH; } while (0)
#define set_stopped_child_used_math(child)        do { (child)->flags |= PF_USED_MATH; } while (0)
#define clear_used_math()                        clear_stopped_child_used_math(current)
#define set_used_math()                                set_stopped_child_used_math(current)

#define conditional_stopped_child_used_math(condition, child) \
        do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0)

#define conditional_used_math(condition)        conditional_stopped_child_used_math(condition, current)

#define copy_to_stopped_child_used_math(child) \
        do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0)

/* NOTE: this will return 0 or PF_USED_MATH, it will never return 1 */
#define tsk_used_math(p)                        ((p)->flags & PF_USED_MATH)
#define used_math()                                tsk_used_math(current)

static __always_inline bool is_percpu_thread(void)
{
#ifdef CONFIG_SMP
        return (current->flags & PF_NO_SETAFFINITY) &&
                (current->nr_cpus_allowed  == 1);
#else
        return true;
#endif
}

/* Per-process atomic flags. */
#define PFA_NO_NEW_PRIVS                0        /* May not gain new privileges. */
#define PFA_SPREAD_PAGE                        1        /* Spread page cache over cpuset */
#define PFA_SPREAD_SLAB                        2        /* Spread some slab caches over cpuset */
#define PFA_SPEC_SSB_DISABLE                3        /* Speculative Store Bypass disabled */
#define PFA_SPEC_SSB_FORCE_DISABLE        4        /* Speculative Store Bypass force disabled*/
#define PFA_SPEC_IB_DISABLE                5        /* Indirect branch speculation restricted */
#define PFA_SPEC_IB_FORCE_DISABLE        6        /* Indirect branch speculation permanently restricted */
#define PFA_SPEC_SSB_NOEXEC                7        /* Speculative Store Bypass clear on execve() */

#define TASK_PFA_TEST(name, func)                                        \
        static inline bool task_##func(struct task_struct *p)                \
        { return test_bit(PFA_##name, &p->atomic_flags); }

#define TASK_PFA_SET(name, func)                                        \
        static inline void task_set_##func(struct task_struct *p)        \
        { set_bit(PFA_##name, &p->atomic_flags); }

#define TASK_PFA_CLEAR(name, func)                                        \
        static inline void task_clear_##func(struct task_struct *p)        \
        { clear_bit(PFA_##name, &p->atomic_flags); }

TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs)
TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs)

TASK_PFA_TEST(SPREAD_PAGE, spread_page)
TASK_PFA_SET(SPREAD_PAGE, spread_page)
TASK_PFA_CLEAR(SPREAD_PAGE, spread_page)

TASK_PFA_TEST(SPREAD_SLAB, spread_slab)
TASK_PFA_SET(SPREAD_SLAB, spread_slab)
TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab)

TASK_PFA_TEST(SPEC_SSB_DISABLE, spec_ssb_disable)
TASK_PFA_SET(SPEC_SSB_DISABLE, spec_ssb_disable)
TASK_PFA_CLEAR(SPEC_SSB_DISABLE, spec_ssb_disable)

TASK_PFA_TEST(SPEC_SSB_NOEXEC, spec_ssb_noexec)
TASK_PFA_SET(SPEC_SSB_NOEXEC, spec_ssb_noexec)
TASK_PFA_CLEAR(SPEC_SSB_NOEXEC, spec_ssb_noexec)

TASK_PFA_TEST(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
TASK_PFA_SET(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)

TASK_PFA_TEST(SPEC_IB_DISABLE, spec_ib_disable)
TASK_PFA_SET(SPEC_IB_DISABLE, spec_ib_disable)
TASK_PFA_CLEAR(SPEC_IB_DISABLE, spec_ib_disable)

TASK_PFA_TEST(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)
TASK_PFA_SET(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)

static inline void
current_restore_flags(unsigned long orig_flags, unsigned long flags)
{
        current->flags &= ~flags;
        current->flags |= orig_flags & flags;
}

extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
extern int task_can_attach(struct task_struct *p);
extern int dl_bw_alloc(int cpu, u64 dl_bw);
extern void dl_bw_free(int cpu, u64 dl_bw);
#ifdef CONFIG_SMP

/* do_set_cpus_allowed() - consider using set_cpus_allowed_ptr() instead */
extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask);

/**
 * set_cpus_allowed_ptr - set CPU affinity mask of a task
 * @p: the task
 * @new_mask: CPU affinity mask
 *
 * Return: zero if successful, or a negative error code
 */
extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask);
extern int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, int node);
extern void release_user_cpus_ptr(struct task_struct *p);
extern int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask);
extern void force_compatible_cpus_allowed_ptr(struct task_struct *p);
extern void relax_compatible_cpus_allowed_ptr(struct task_struct *p);
#else
static inline void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
{
}
static inline int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
{
        /* Opencoded cpumask_test_cpu(0, new_mask) to avoid dependency on cpumask.h */
        if ((*cpumask_bits(new_mask) & 1) == 0)
                return -EINVAL;
        return 0;
}
static inline int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, int node)
{
        if (src->user_cpus_ptr)
                return -EINVAL;
        return 0;
}
static inline void release_user_cpus_ptr(struct task_struct *p)
{
        WARN_ON(p->user_cpus_ptr);
}

static inline int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask)
{
        return 0;
}
#endif

extern int yield_to(struct task_struct *p, bool preempt);
extern void set_user_nice(struct task_struct *p, long nice);
extern int task_prio(const struct task_struct *p);

/**
 * task_nice - return the nice value of a given task.
 * @p: the task in question.
 *
 * Return: The nice value [ -20 ... 0 ... 19 ].
 */
static inline int task_nice(const struct task_struct *p)
{
        return PRIO_TO_NICE((p)->static_prio);
}

extern int can_nice(const struct task_struct *p, const int nice);
extern int task_curr(const struct task_struct *p);
extern int idle_cpu(int cpu);
extern int available_idle_cpu(int cpu);
extern int sched_setscheduler(struct task_struct *, int, const struct sched_param *);
extern int sched_setscheduler_nocheck(struct task_struct *, int, const struct sched_param *);
extern void sched_set_fifo(struct task_struct *p);
extern void sched_set_fifo_low(struct task_struct *p);
extern void sched_set_normal(struct task_struct *p, int nice);
extern int sched_setattr(struct task_struct *, const struct sched_attr *);
extern int sched_setattr_nocheck(struct task_struct *, const struct sched_attr *);
extern struct task_struct *idle_task(int cpu);

/**
 * is_idle_task - is the specified task an idle task?
 * @p: the task in question.
 *
 * Return: 1 if @p is an idle task. 0 otherwise.
 */
static __always_inline bool is_idle_task(const struct task_struct *p)
{
        return !!(p->flags & PF_IDLE);
}

extern struct task_struct *curr_task(int cpu);
extern void ia64_set_curr_task(int cpu, struct task_struct *p);

void yield(void);

union thread_union {
        struct task_struct task;
#ifndef CONFIG_THREAD_INFO_IN_TASK
        struct thread_info thread_info;
#endif
        unsigned long stack[THREAD_SIZE/sizeof(long)];
};

#ifndef CONFIG_THREAD_INFO_IN_TASK
extern struct thread_info init_thread_info;
#endif

extern unsigned long init_stack[THREAD_SIZE / sizeof(unsigned long)];

#ifdef CONFIG_THREAD_INFO_IN_TASK
# define task_thread_info(task)        (&(task)->thread_info)
#else
# define task_thread_info(task)        ((struct thread_info *)(task)->stack)
#endif

/*
 * find a task by one of its numerical ids
 *
 * find_task_by_pid_ns():
 *      finds a task by its pid in the specified namespace
 * find_task_by_vpid():
 *      finds a task by its virtual pid
 *
 * see also find_vpid() etc in include/linux/pid.h
 */

extern struct task_struct *find_task_by_vpid(pid_t nr);
extern struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns);

/*
 * find a task by its virtual pid and get the task struct
 */
extern struct task_struct *find_get_task_by_vpid(pid_t nr);

extern int wake_up_state(struct task_struct *tsk, unsigned int state);
extern int wake_up_process(struct task_struct *tsk);
extern void wake_up_new_task(struct task_struct *tsk);

#ifdef CONFIG_SMP
extern void kick_process(struct task_struct *tsk);
#else
static inline void kick_process(struct task_struct *tsk) { }
#endif

extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec);
#define set_task_comm(tsk, from) ({                        \
        BUILD_BUG_ON(sizeof(from) != TASK_COMM_LEN);        \
        __set_task_comm(tsk, from, false);                \
})

/*
 * - Why not use task_lock()?
 *   User space can randomly change their names anyway, so locking for readers
 *   doesn't make sense. For writers, locking is probably necessary, as a race
 *   condition could lead to long-term mixed results.
 *   The strscpy_pad() in __set_task_comm() can ensure that the task comm is
 *   always NUL-terminated and zero-padded. Therefore the race condition between
 *   reader and writer is not an issue.
 *
 * - BUILD_BUG_ON() can help prevent the buf from being truncated.
 *   Since the callers don't perform any return value checks, this safeguard is
 *   necessary.
 */
#define get_task_comm(buf, tsk) ({                        \
        BUILD_BUG_ON(sizeof(buf) < TASK_COMM_LEN);        \
        strscpy_pad(buf, (tsk)->comm);                        \
        buf;                                                \
})

#ifdef CONFIG_SMP
static __always_inline void scheduler_ipi(void)
{
        /*
         * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
         * TIF_NEED_RESCHED remotely (for the first time) will also send
         * this IPI.
         */
        preempt_fold_need_resched();
}
#else
static inline void scheduler_ipi(void) { }
#endif

extern unsigned long wait_task_inactive(struct task_struct *, unsigned int match_state);

/*
 * Set thread flags in other task's structures.
 * See asm/thread_info.h for TIF_xxxx flags available:
 */
static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag)
{
        set_ti_thread_flag(task_thread_info(tsk), flag);
}

static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag)
{
        clear_ti_thread_flag(task_thread_info(tsk), flag);
}

static inline void update_tsk_thread_flag(struct task_struct *tsk, int flag,
                                          bool value)
{
        update_ti_thread_flag(task_thread_info(tsk), flag, value);
}

static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag)
{
        return test_and_set_ti_thread_flag(task_thread_info(tsk), flag);
}

static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag)
{
        return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag);
}

static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag)
{
        return test_ti_thread_flag(task_thread_info(tsk), flag);
}

static inline void set_tsk_need_resched(struct task_struct *tsk)
{
        set_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
}

static inline void clear_tsk_need_resched(struct task_struct *tsk)
{
        atomic_long_andnot(_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY,
                           (atomic_long_t *)&task_thread_info(tsk)->flags);
}

static inline int test_tsk_need_resched(struct task_struct *tsk)
{
        return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
}

/*
 * cond_resched() and cond_resched_lock(): latency reduction via
 * explicit rescheduling in places that are safe. The return
 * value indicates whether a reschedule was done in fact.
 * cond_resched_lock() will drop the spinlock before scheduling,
 */
#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC)
extern int __cond_resched(void);

#if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)

void sched_dynamic_klp_enable(void);
void sched_dynamic_klp_disable(void);

DECLARE_STATIC_CALL(cond_resched, __cond_resched);

static __always_inline int _cond_resched(void)
{
        return static_call_mod(cond_resched)();
}

#elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)

extern int dynamic_cond_resched(void);

static __always_inline int _cond_resched(void)
{
        return dynamic_cond_resched();
}

#else /* !CONFIG_PREEMPTION */

static inline int _cond_resched(void)
{
        klp_sched_try_switch();
        return __cond_resched();
}

#endif /* PREEMPT_DYNAMIC && CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */

#else /* CONFIG_PREEMPTION && !CONFIG_PREEMPT_DYNAMIC */

static inline int _cond_resched(void)
{
        klp_sched_try_switch();
        return 0;
}

#endif /* !CONFIG_PREEMPTION || CONFIG_PREEMPT_DYNAMIC */

#define cond_resched() ({                        \
        __might_resched(__FILE__, __LINE__, 0);        \
        _cond_resched();                        \
})

extern int __cond_resched_lock(spinlock_t *lock);
extern int __cond_resched_rwlock_read(rwlock_t *lock);
extern int __cond_resched_rwlock_write(rwlock_t *lock);

#define MIGHT_RESCHED_RCU_SHIFT                8
#define MIGHT_RESCHED_PREEMPT_MASK        ((1U << MIGHT_RESCHED_RCU_SHIFT) - 1)

#ifndef CONFIG_PREEMPT_RT
/*
 * Non RT kernels have an elevated preempt count due to the held lock,
 * but are not allowed to be inside a RCU read side critical section
 */
# define PREEMPT_LOCK_RESCHED_OFFSETS        PREEMPT_LOCK_OFFSET
#else
/*
 * spin/rw_lock() on RT implies rcu_read_lock(). The might_sleep() check in
 * cond_resched*lock() has to take that into account because it checks for
 * preempt_count() and rcu_preempt_depth().
 */
# define PREEMPT_LOCK_RESCHED_OFFSETS        \
        (PREEMPT_LOCK_OFFSET + (1U << MIGHT_RESCHED_RCU_SHIFT))
#endif

#define cond_resched_lock(lock) ({                                                \
        __might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS);        \
        __cond_resched_lock(lock);                                                \
})

#define cond_resched_rwlock_read(lock) ({                                        \
        __might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS);        \
        __cond_resched_rwlock_read(lock);                                        \
})

#define cond_resched_rwlock_write(lock) ({                                        \
        __might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS);        \
        __cond_resched_rwlock_write(lock);                                        \
})

static __always_inline bool need_resched(void)
{
        return unlikely(tif_need_resched());
}

/*
 * Wrappers for p->thread_info->cpu access. No-op on UP.
 */
#ifdef CONFIG_SMP

static inline unsigned int task_cpu(const struct task_struct *p)
{
        return READ_ONCE(task_thread_info(p)->cpu);
}

extern void set_task_cpu(struct task_struct *p, unsigned int cpu);

#else

static inline unsigned int task_cpu(const struct task_struct *p)
{
        return 0;
}

static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
{
}

#endif /* CONFIG_SMP */

static inline bool task_is_runnable(struct task_struct *p)
{
        return p->on_rq && !p->se.sched_delayed;
}

extern bool sched_task_on_rq(struct task_struct *p);
extern unsigned long get_wchan(struct task_struct *p);
extern struct task_struct *cpu_curr_snapshot(int cpu);

#include <linux/spinlock.h>

/*
 * In order to reduce various lock holder preemption latencies provide an
 * interface to see if a vCPU is currently running or not.
 *
 * This allows us to terminate optimistic spin loops and block, analogous to
 * the native optimistic spin heuristic of testing if the lock owner task is
 * running or not.
 */
#ifndef vcpu_is_preempted
static inline bool vcpu_is_preempted(int cpu)
{
        return false;
}
#endif

extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
extern long sched_getaffinity(pid_t pid, struct cpumask *mask);

#ifndef TASK_SIZE_OF
#define TASK_SIZE_OF(tsk)        TASK_SIZE
#endif

#ifdef CONFIG_SMP
static inline bool owner_on_cpu(struct task_struct *owner)
{
        /*
         * As lock holder preemption issue, we both skip spinning if
         * task is not on cpu or its cpu is preempted
         */
        return READ_ONCE(owner->on_cpu) && !vcpu_is_preempted(task_cpu(owner));
}

/* Returns effective CPU energy utilization, as seen by the scheduler */
unsigned long sched_cpu_util(int cpu);
#endif /* CONFIG_SMP */

#ifdef CONFIG_SCHED_CORE
extern void sched_core_free(struct task_struct *tsk);
extern void sched_core_fork(struct task_struct *p);
extern int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type,
                                unsigned long uaddr);
extern int sched_core_idle_cpu(int cpu);
#else
static inline void sched_core_free(struct task_struct *tsk) { }
static inline void sched_core_fork(struct task_struct *p) { }
static inline int sched_core_idle_cpu(int cpu) { return idle_cpu(cpu); }
#endif

extern void sched_set_stop_task(int cpu, struct task_struct *stop);

#ifdef CONFIG_MEM_ALLOC_PROFILING
static __always_inline struct alloc_tag *alloc_tag_save(struct alloc_tag *tag)
{
        swap(current->alloc_tag, tag);
        return tag;
}

static __always_inline void alloc_tag_restore(struct alloc_tag *tag, struct alloc_tag *old)
{
#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
        WARN(current->alloc_tag != tag, "current->alloc_tag was changed:\n");
#endif
        current->alloc_tag = old;
}
#else
#define alloc_tag_save(_tag)                        NULL
#define alloc_tag_restore(_tag, _old)                do {} while (0)
#endif

#endif



































































































































 1024 


 1026 






































































































































































































































































































































































































































































































































































































































































































































    4 



    2 


    2 





















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *
 * Copyright (C) Alan Cox GW4PTS (alan@lxorguk.ukuu.org.uk)
 * Copyright (C) Jonathan Naylor G4KLX (g4klx@g4klx.demon.co.uk)
 * Copyright (C) Darryl Miles G7LED (dlm@g7led.demon.co.uk)
 * Copyright (C) Steven Whitehouse GW7RRM (stevew@acm.org)
 * Copyright (C) Joerg Reuter DL1BKE (jreuter@yaina.de)
 * Copyright (C) Hans-Joachim Hetscher DD8NE (dd8ne@bnv-bamberg.de)
 * Copyright (C) Hans Alblas PE1AYX (hans@esrac.ele.tue.nl)
 * Copyright (C) Frederic Rible F1OAT (frible@teaser.fr)
 */
#include <linux/capability.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/socket.h>
#include <linux/in.h>
#include <linux/kernel.h>
#include <linux/sched/signal.h>
#include <linux/timer.h>
#include <linux/string.h>
#include <linux/sockios.h>
#include <linux/net.h>
#include <linux/slab.h>
#include <net/ax25.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/if_arp.h>
#include <linux/skbuff.h>
#include <net/sock.h>
#include <linux/uaccess.h>
#include <linux/fcntl.h>
#include <linux/termios.h>        /* For TIOCINQ/OUTQ */
#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/notifier.h>
#include <linux/proc_fs.h>
#include <linux/stat.h>
#include <linux/sysctl.h>
#include <linux/init.h>
#include <linux/spinlock.h>
#include <net/net_namespace.h>
#include <net/tcp_states.h>
#include <net/ip.h>
#include <net/arp.h>



HLIST_HEAD(ax25_list);
DEFINE_SPINLOCK(ax25_list_lock);

static const struct proto_ops ax25_proto_ops;

static void ax25_free_sock(struct sock *sk)
{
        ax25_cb_put(sk_to_ax25(sk));
}

/*
 *        Socket removal during an interrupt is now safe.
 */
static void ax25_cb_del(ax25_cb *ax25)
{
        spin_lock_bh(&ax25_list_lock);
        if (!hlist_unhashed(&ax25->ax25_node)) {
                hlist_del_init(&ax25->ax25_node);
                ax25_cb_put(ax25);
        }
        spin_unlock_bh(&ax25_list_lock);
}

/*
 *        Kill all bound sockets on a dropped device.
 */
static void ax25_kill_by_device(struct net_device *dev)
{
        ax25_dev *ax25_dev;
        ax25_cb *s;
        struct sock *sk;

        if ((ax25_dev = ax25_dev_ax25dev(dev)) == NULL)
                return;
        ax25_dev->device_up = false;

        spin_lock_bh(&ax25_list_lock);
again:
        ax25_for_each(s, &ax25_list) {
                if (s->ax25_dev == ax25_dev) {
                        sk = s->sk;
                        if (!sk) {
                                spin_unlock_bh(&ax25_list_lock);
                                ax25_disconnect(s, ENETUNREACH);
                                s->ax25_dev = NULL;
                                ax25_cb_del(s);
                                spin_lock_bh(&ax25_list_lock);
                                goto again;
                        }
                        sock_hold(sk);
                        spin_unlock_bh(&ax25_list_lock);
                        lock_sock(sk);
                        ax25_disconnect(s, ENETUNREACH);
                        s->ax25_dev = NULL;
                        if (sk->sk_socket) {
                                netdev_put(ax25_dev->dev,
                                           &s->dev_tracker);
                                ax25_dev_put(ax25_dev);
                        }
                        ax25_cb_del(s);
                        release_sock(sk);
                        spin_lock_bh(&ax25_list_lock);
                        sock_put(sk);
                        /* The entry could have been deleted from the
                         * list meanwhile and thus the next pointer is
                         * no longer valid.  Play it safe and restart
                         * the scan.  Forward progress is ensured
                         * because we set s->ax25_dev to NULL and we
                         * are never passed a NULL 'dev' argument.
                         */
                        goto again;
                }
        }
        spin_unlock_bh(&ax25_list_lock);
}

/*
 *        Handle device status changes.
 */
static int ax25_device_event(struct notifier_block *this, unsigned long event,
                             void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);

        if (!net_eq(dev_net(dev), &init_net))
                return NOTIFY_DONE;

        /* Reject non AX.25 devices */
        if (dev->type != ARPHRD_AX25)
                return NOTIFY_DONE;

        switch (event) {
        case NETDEV_UP:
                ax25_dev_device_up(dev);
                break;
        case NETDEV_DOWN:
                ax25_kill_by_device(dev);
                ax25_rt_device_down(dev);
                ax25_dev_device_down(dev);
                break;
        default:
                break;
        }

        return NOTIFY_DONE;
}

/*
 *        Add a socket to the bound sockets list.
 */
void ax25_cb_add(ax25_cb *ax25)
{
        spin_lock_bh(&ax25_list_lock);
        ax25_cb_hold(ax25);
        hlist_add_head(&ax25->ax25_node, &ax25_list);
        spin_unlock_bh(&ax25_list_lock);
}

/*
 *        Find a socket that wants to accept the SABM we have just
 *        received.
 */
struct sock *ax25_find_listener(ax25_address *addr, int digi,
        struct net_device *dev, int type)
{
        ax25_cb *s;

        spin_lock(&ax25_list_lock);
        ax25_for_each(s, &ax25_list) {
                if ((s->iamdigi && !digi) || (!s->iamdigi && digi))
                        continue;
                if (s->sk && !ax25cmp(&s->source_addr, addr) &&
                    s->sk->sk_type == type && s->sk->sk_state == TCP_LISTEN) {
                        /* If device is null we match any device */
                        if (s->ax25_dev == NULL || s->ax25_dev->dev == dev) {
                                sock_hold(s->sk);
                                spin_unlock(&ax25_list_lock);
                                return s->sk;
                        }
                }
        }
        spin_unlock(&ax25_list_lock);

        return NULL;
}

/*
 *        Find an AX.25 socket given both ends.
 */
struct sock *ax25_get_socket(ax25_address *my_addr, ax25_address *dest_addr,
        int type)
{
        struct sock *sk = NULL;
        ax25_cb *s;

        spin_lock(&ax25_list_lock);
        ax25_for_each(s, &ax25_list) {
                if (s->sk && !ax25cmp(&s->source_addr, my_addr) &&
                    !ax25cmp(&s->dest_addr, dest_addr) &&
                    s->sk->sk_type == type) {
                        sk = s->sk;
                        sock_hold(sk);
                        break;
                }
        }

        spin_unlock(&ax25_list_lock);

        return sk;
}

/*
 *        Find an AX.25 control block given both ends. It will only pick up
 *        floating AX.25 control blocks or non Raw socket bound control blocks.
 */
ax25_cb *ax25_find_cb(const ax25_address *src_addr, ax25_address *dest_addr,
        ax25_digi *digi, struct net_device *dev)
{
        ax25_cb *s;

        spin_lock_bh(&ax25_list_lock);
        ax25_for_each(s, &ax25_list) {
                if (s->sk && s->sk->sk_type != SOCK_SEQPACKET)
                        continue;
                if (s->ax25_dev == NULL)
                        continue;
                if (ax25cmp(&s->source_addr, src_addr) == 0 && ax25cmp(&s->dest_addr, dest_addr) == 0 && s->ax25_dev->dev == dev) {
                        if (digi != NULL && digi->ndigi != 0) {
                                if (s->digipeat == NULL)
                                        continue;
                                if (ax25digicmp(s->digipeat, digi) != 0)
                                        continue;
                        } else {
                                if (s->digipeat != NULL && s->digipeat->ndigi != 0)
                                        continue;
                        }
                        ax25_cb_hold(s);
                        spin_unlock_bh(&ax25_list_lock);

                        return s;
                }
        }
        spin_unlock_bh(&ax25_list_lock);

        return NULL;
}

EXPORT_SYMBOL(ax25_find_cb);

void ax25_send_to_raw(ax25_address *addr, struct sk_buff *skb, int proto)
{
        ax25_cb *s;
        struct sk_buff *copy;

        spin_lock(&ax25_list_lock);
        ax25_for_each(s, &ax25_list) {
                if (s->sk != NULL && ax25cmp(&s->source_addr, addr) == 0 &&
                    s->sk->sk_type == SOCK_RAW &&
                    s->sk->sk_protocol == proto &&
                    s->ax25_dev->dev == skb->dev &&
                    atomic_read(&s->sk->sk_rmem_alloc) <= s->sk->sk_rcvbuf) {
                        if ((copy = skb_clone(skb, GFP_ATOMIC)) == NULL)
                                continue;
                        if (sock_queue_rcv_skb(s->sk, copy) != 0)
                                kfree_skb(copy);
                }
        }
        spin_unlock(&ax25_list_lock);
}

/*
 *        Deferred destroy.
 */
void ax25_destroy_socket(ax25_cb *);

/*
 *        Handler for deferred kills.
 */
static void ax25_destroy_timer(struct timer_list *t)
{
        ax25_cb *ax25 = from_timer(ax25, t, dtimer);
        struct sock *sk;

        sk=ax25->sk;

        bh_lock_sock(sk);
        sock_hold(sk);
        ax25_destroy_socket(ax25);
        bh_unlock_sock(sk);
        sock_put(sk);
}

/*
 *        This is called from user mode and the timers. Thus it protects itself
 *        against interrupt users but doesn't worry about being called during
 *        work. Once it is removed from the queue no interrupt or bottom half
 *        will touch it and we are (fairly 8-) ) safe.
 */
void ax25_destroy_socket(ax25_cb *ax25)
{
        struct sk_buff *skb;

        ax25_cb_del(ax25);

        ax25_stop_heartbeat(ax25);
        ax25_stop_t1timer(ax25);
        ax25_stop_t2timer(ax25);
        ax25_stop_t3timer(ax25);
        ax25_stop_idletimer(ax25);

        ax25_clear_queues(ax25);        /* Flush the queues */

        if (ax25->sk != NULL) {
                while ((skb = skb_dequeue(&ax25->sk->sk_receive_queue)) != NULL) {
                        if (skb->sk != ax25->sk) {
                                /* A pending connection */
                                ax25_cb *sax25 = sk_to_ax25(skb->sk);

                                /* Queue the unaccepted socket for death */
                                sock_orphan(skb->sk);

                                /* 9A4GL: hack to release unaccepted sockets */
                                skb->sk->sk_state = TCP_LISTEN;

                                ax25_start_heartbeat(sax25);
                                sax25->state = AX25_STATE_0;
                        }

                        kfree_skb(skb);
                }
                skb_queue_purge(&ax25->sk->sk_write_queue);
        }

        if (ax25->sk != NULL) {
                if (sk_has_allocations(ax25->sk)) {
                        /* Defer: outstanding buffers */
                        timer_setup(&ax25->dtimer, ax25_destroy_timer, 0);
                        ax25->dtimer.expires  = jiffies + 2 * HZ;
                        add_timer(&ax25->dtimer);
                } else {
                        struct sock *sk=ax25->sk;
                        ax25->sk=NULL;
                        sock_put(sk);
                }
        } else {
                ax25_cb_put(ax25);
        }
}

/*
 * dl1bke 960311: set parameters for existing AX.25 connections,
 *                  includes a KILL command to abort any connection.
 *                  VERY useful for debugging ;-)
 */
static int ax25_ctl_ioctl(const unsigned int cmd, void __user *arg)
{
        struct ax25_ctl_struct ax25_ctl;
        ax25_digi digi;
        ax25_dev *ax25_dev;
        ax25_cb *ax25;
        unsigned int k;
        int ret = 0;

        if (copy_from_user(&ax25_ctl, arg, sizeof(ax25_ctl)))
                return -EFAULT;

        if (ax25_ctl.digi_count > AX25_MAX_DIGIS)
                return -EINVAL;

        if (ax25_ctl.arg > ULONG_MAX / HZ && ax25_ctl.cmd != AX25_KILL)
                return -EINVAL;

        ax25_dev = ax25_addr_ax25dev(&ax25_ctl.port_addr);
        if (!ax25_dev)
                return -ENODEV;

        digi.ndigi = ax25_ctl.digi_count;
        for (k = 0; k < digi.ndigi; k++)
                digi.calls[k] = ax25_ctl.digi_addr[k];

        ax25 = ax25_find_cb(&ax25_ctl.source_addr, &ax25_ctl.dest_addr, &digi, ax25_dev->dev);
        if (!ax25) {
                ax25_dev_put(ax25_dev);
                return -ENOTCONN;
        }

        switch (ax25_ctl.cmd) {
        case AX25_KILL:
                ax25_send_control(ax25, AX25_DISC, AX25_POLLON, AX25_COMMAND);
#ifdef CONFIG_AX25_DAMA_SLAVE
                if (ax25_dev->dama.slave && ax25->ax25_dev->values[AX25_VALUES_PROTOCOL] == AX25_PROTO_DAMA_SLAVE)
                        ax25_dama_off(ax25);
#endif
                ax25_disconnect(ax25, ENETRESET);
                break;

        case AX25_WINDOW:
                if (ax25->modulus == AX25_MODULUS) {
                        if (ax25_ctl.arg < 1 || ax25_ctl.arg > 7)
                                goto einval_put;
                } else {
                        if (ax25_ctl.arg < 1 || ax25_ctl.arg > 63)
                                goto einval_put;
                }
                ax25->window = ax25_ctl.arg;
                break;

        case AX25_T1:
                if (ax25_ctl.arg < 1 || ax25_ctl.arg > ULONG_MAX / HZ)
                        goto einval_put;
                ax25->rtt = (ax25_ctl.arg * HZ) / 2;
                ax25->t1  = ax25_ctl.arg * HZ;
                break;

        case AX25_T2:
                if (ax25_ctl.arg < 1 || ax25_ctl.arg > ULONG_MAX / HZ)
                        goto einval_put;
                ax25->t2 = ax25_ctl.arg * HZ;
                break;

        case AX25_N2:
                if (ax25_ctl.arg < 1 || ax25_ctl.arg > 31)
                        goto einval_put;
                ax25->n2count = 0;
                ax25->n2 = ax25_ctl.arg;
                break;

        case AX25_T3:
                if (ax25_ctl.arg > ULONG_MAX / HZ)
                        goto einval_put;
                ax25->t3 = ax25_ctl.arg * HZ;
                break;

        case AX25_IDLE:
                if (ax25_ctl.arg > ULONG_MAX / (60 * HZ))
                        goto einval_put;

                ax25->idle = ax25_ctl.arg * 60 * HZ;
                break;

        case AX25_PACLEN:
                if (ax25_ctl.arg < 16 || ax25_ctl.arg > 65535)
                        goto einval_put;
                ax25->paclen = ax25_ctl.arg;
                break;

        default:
                goto einval_put;
          }

out_put:
        ax25_dev_put(ax25_dev);
        ax25_cb_put(ax25);
        return ret;

einval_put:
        ret = -EINVAL;
        goto out_put;
}

static void ax25_fillin_cb_from_dev(ax25_cb *ax25, const ax25_dev *ax25_dev)
{
        ax25->rtt     = msecs_to_jiffies(ax25_dev->values[AX25_VALUES_T1]) / 2;
        ax25->t1      = msecs_to_jiffies(ax25_dev->values[AX25_VALUES_T1]);
        ax25->t2      = msecs_to_jiffies(ax25_dev->values[AX25_VALUES_T2]);
        ax25->t3      = msecs_to_jiffies(ax25_dev->values[AX25_VALUES_T3]);
        ax25->n2      = ax25_dev->values[AX25_VALUES_N2];
        ax25->paclen  = ax25_dev->values[AX25_VALUES_PACLEN];
        ax25->idle    = msecs_to_jiffies(ax25_dev->values[AX25_VALUES_IDLE]);
        ax25->backoff = ax25_dev->values[AX25_VALUES_BACKOFF];

        if (ax25_dev->values[AX25_VALUES_AXDEFMODE]) {
                ax25->modulus = AX25_EMODULUS;
                ax25->window  = ax25_dev->values[AX25_VALUES_EWINDOW];
        } else {
                ax25->modulus = AX25_MODULUS;
                ax25->window  = ax25_dev->values[AX25_VALUES_WINDOW];
        }
}

/*
 *        Fill in a created AX.25 created control block with the default
 *        values for a particular device.
 */
void ax25_fillin_cb(ax25_cb *ax25, ax25_dev *ax25_dev)
{
        ax25->ax25_dev = ax25_dev;

        if (ax25->ax25_dev != NULL) {
                ax25_fillin_cb_from_dev(ax25, ax25_dev);
                return;
        }

        /*
         * No device, use kernel / AX.25 spec default values
         */
        ax25->rtt     = msecs_to_jiffies(AX25_DEF_T1) / 2;
        ax25->t1      = msecs_to_jiffies(AX25_DEF_T1);
        ax25->t2      = msecs_to_jiffies(AX25_DEF_T2);
        ax25->t3      = msecs_to_jiffies(AX25_DEF_T3);
        ax25->n2      = AX25_DEF_N2;
        ax25->paclen  = AX25_DEF_PACLEN;
        ax25->idle    = msecs_to_jiffies(AX25_DEF_IDLE);
        ax25->backoff = AX25_DEF_BACKOFF;

        if (AX25_DEF_AXDEFMODE) {
                ax25->modulus = AX25_EMODULUS;
                ax25->window  = AX25_DEF_EWINDOW;
        } else {
                ax25->modulus = AX25_MODULUS;
                ax25->window  = AX25_DEF_WINDOW;
        }
}

/*
 * Create an empty AX.25 control block.
 */
ax25_cb *ax25_create_cb(void)
{
        ax25_cb *ax25;

        if ((ax25 = kzalloc(sizeof(*ax25), GFP_ATOMIC)) == NULL)
                return NULL;

        refcount_set(&ax25->refcount, 1);

        skb_queue_head_init(&ax25->write_queue);
        skb_queue_head_init(&ax25->frag_queue);
        skb_queue_head_init(&ax25->ack_queue);
        skb_queue_head_init(&ax25->reseq_queue);

        ax25_setup_timers(ax25);

        ax25_fillin_cb(ax25, NULL);

        ax25->state = AX25_STATE_0;

        return ax25;
}

/*
 *        Handling for system calls applied via the various interfaces to an
 *        AX25 socket object
 */

static int ax25_setsockopt(struct socket *sock, int level, int optname,
                sockptr_t optval, unsigned int optlen)
{
        struct sock *sk = sock->sk;
        ax25_cb *ax25;
        struct net_device *dev;
        char devname[IFNAMSIZ];
        unsigned int opt;
        int res = 0;

        if (level != SOL_AX25)
                return -ENOPROTOOPT;

        if (optlen < sizeof(unsigned int))
                return -EINVAL;

        if (copy_from_sockptr(&opt, optval, sizeof(unsigned int)))
                return -EFAULT;

        lock_sock(sk);
        ax25 = sk_to_ax25(sk);

        switch (optname) {
        case AX25_WINDOW:
                if (ax25->modulus == AX25_MODULUS) {
                        if (opt < 1 || opt > 7) {
                                res = -EINVAL;
                                break;
                        }
                } else {
                        if (opt < 1 || opt > 63) {
                                res = -EINVAL;
                                break;
                        }
                }
                ax25->window = opt;
                break;

        case AX25_T1:
                if (opt < 1 || opt > UINT_MAX / HZ) {
                        res = -EINVAL;
                        break;
                }
                ax25->rtt = (opt * HZ) >> 1;
                ax25->t1  = opt * HZ;
                break;

        case AX25_T2:
                if (opt < 1 || opt > UINT_MAX / HZ) {
                        res = -EINVAL;
                        break;
                }
                ax25->t2 = opt * HZ;
                break;

        case AX25_N2:
                if (opt < 1 || opt > 31) {
                        res = -EINVAL;
                        break;
                }
                ax25->n2 = opt;
                break;

        case AX25_T3:
                if (opt < 1 || opt > UINT_MAX / HZ) {
                        res = -EINVAL;
                        break;
                }
                ax25->t3 = opt * HZ;
                break;

        case AX25_IDLE:
                if (opt > UINT_MAX / (60 * HZ)) {
                        res = -EINVAL;
                        break;
                }
                ax25->idle = opt * 60 * HZ;
                break;

        case AX25_BACKOFF:
                if (opt > 2) {
                        res = -EINVAL;
                        break;
                }
                ax25->backoff = opt;
                break;

        case AX25_EXTSEQ:
                ax25->modulus = opt ? AX25_EMODULUS : AX25_MODULUS;
                break;

        case AX25_PIDINCL:
                ax25->pidincl = opt ? 1 : 0;
                break;

        case AX25_IAMDIGI:
                ax25->iamdigi = opt ? 1 : 0;
                break;

        case AX25_PACLEN:
                if (opt < 16 || opt > 65535) {
                        res = -EINVAL;
                        break;
                }
                ax25->paclen = opt;
                break;

        case SO_BINDTODEVICE:
                if (optlen > IFNAMSIZ - 1)
                        optlen = IFNAMSIZ - 1;

                memset(devname, 0, sizeof(devname));

                if (copy_from_sockptr(devname, optval, optlen)) {
                        res = -EFAULT;
                        break;
                }

                if (sk->sk_type == SOCK_SEQPACKET &&
                   (sock->state != SS_UNCONNECTED ||
                    sk->sk_state == TCP_LISTEN)) {
                        res = -EADDRNOTAVAIL;
                        break;
                }

                rcu_read_lock();
                dev = dev_get_by_name_rcu(&init_net, devname);
                if (!dev) {
                        rcu_read_unlock();
                        res = -ENODEV;
                        break;
                }

                if (ax25->ax25_dev) {
                        if (dev == ax25->ax25_dev->dev) {
                                rcu_read_unlock();
                                break;
                        }
                        netdev_put(ax25->ax25_dev->dev, &ax25->dev_tracker);
                        ax25_dev_put(ax25->ax25_dev);
                }

                ax25->ax25_dev = ax25_dev_ax25dev(dev);
                if (!ax25->ax25_dev) {
                        rcu_read_unlock();
                        res = -ENODEV;
                        break;
                }
                ax25_fillin_cb(ax25, ax25->ax25_dev);
                netdev_hold(dev, &ax25->dev_tracker, GFP_ATOMIC);
                ax25_dev_hold(ax25->ax25_dev);
                rcu_read_unlock();
                break;

        default:
                res = -ENOPROTOOPT;
        }
        release_sock(sk);

        return res;
}

static int ax25_getsockopt(struct socket *sock, int level, int optname,
        char __user *optval, int __user *optlen)
{
        struct sock *sk = sock->sk;
        ax25_cb *ax25;
        struct ax25_dev *ax25_dev;
        char devname[IFNAMSIZ];
        void *valptr;
        int val = 0;
        int maxlen, length;

        if (level != SOL_AX25)
                return -ENOPROTOOPT;

        if (get_user(maxlen, optlen))
                return -EFAULT;

        if (maxlen < 1)
                return -EFAULT;

        valptr = &val;
        length = min_t(unsigned int, maxlen, sizeof(int));

        lock_sock(sk);
        ax25 = sk_to_ax25(sk);

        switch (optname) {
        case AX25_WINDOW:
                val = ax25->window;
                break;

        case AX25_T1:
                val = ax25->t1 / HZ;
                break;

        case AX25_T2:
                val = ax25->t2 / HZ;
                break;

        case AX25_N2:
                val = ax25->n2;
                break;

        case AX25_T3:
                val = ax25->t3 / HZ;
                break;

        case AX25_IDLE:
                val = ax25->idle / (60 * HZ);
                break;

        case AX25_BACKOFF:
                val = ax25->backoff;
                break;

        case AX25_EXTSEQ:
                val = (ax25->modulus == AX25_EMODULUS);
                break;

        case AX25_PIDINCL:
                val = ax25->pidincl;
                break;

        case AX25_IAMDIGI:
                val = ax25->iamdigi;
                break;

        case AX25_PACLEN:
                val = ax25->paclen;
                break;

        case SO_BINDTODEVICE:
                ax25_dev = ax25->ax25_dev;

                if (ax25_dev != NULL && ax25_dev->dev != NULL) {
                        strscpy(devname, ax25_dev->dev->name, sizeof(devname));
                        length = strlen(devname) + 1;
                } else {
                        *devname = '\0';
                        length = 1;
                }

                valptr = devname;
                break;

        default:
                release_sock(sk);
                return -ENOPROTOOPT;
        }
        release_sock(sk);

        if (put_user(length, optlen))
                return -EFAULT;

        return copy_to_user(optval, valptr, length) ? -EFAULT : 0;
}

static int ax25_listen(struct socket *sock, int backlog)
{
        struct sock *sk = sock->sk;
        int res = 0;

        lock_sock(sk);
        if (sk->sk_type == SOCK_SEQPACKET && sk->sk_state != TCP_LISTEN) {
                sk->sk_max_ack_backlog = backlog;
                sk->sk_state           = TCP_LISTEN;
                goto out;
        }
        res = -EOPNOTSUPP;

out:
        release_sock(sk);

        return res;
}

/*
 * XXX: when creating ax25_sock we should update the .obj_size setting
 * below.
 */
static struct proto ax25_proto = {
        .name          = "AX25",
        .owner          = THIS_MODULE,
        .obj_size = sizeof(struct ax25_sock),
};

static int ax25_create(struct net *net, struct socket *sock, int protocol,
                       int kern)
{
        struct sock *sk;
        ax25_cb *ax25;

        if (protocol < 0 || protocol > U8_MAX)
                return -EINVAL;

        if (!net_eq(net, &init_net))
                return -EAFNOSUPPORT;

        switch (sock->type) {
        case SOCK_DGRAM:
                if (protocol == 0 || protocol == PF_AX25)
                        protocol = AX25_P_TEXT;
                break;

        case SOCK_SEQPACKET:
                switch (protocol) {
                case 0:
                case PF_AX25:        /* For CLX */
                        protocol = AX25_P_TEXT;
                        break;
                case AX25_P_SEGMENT:
#ifdef CONFIG_INET
                case AX25_P_ARP:
                case AX25_P_IP:
#endif
#ifdef CONFIG_NETROM
                case AX25_P_NETROM:
#endif
#ifdef CONFIG_ROSE
                case AX25_P_ROSE:
#endif
                        return -ESOCKTNOSUPPORT;
#ifdef CONFIG_NETROM_MODULE
                case AX25_P_NETROM:
                        if (ax25_protocol_is_registered(AX25_P_NETROM))
                                return -ESOCKTNOSUPPORT;
                        break;
#endif
#ifdef CONFIG_ROSE_MODULE
                case AX25_P_ROSE:
                        if (ax25_protocol_is_registered(AX25_P_ROSE))
                                return -ESOCKTNOSUPPORT;
                        break;
#endif
                default:
                        break;
                }
                break;

        case SOCK_RAW:
                if (!capable(CAP_NET_RAW))
                        return -EPERM;
                break;
        default:
                return -ESOCKTNOSUPPORT;
        }

        sk = sk_alloc(net, PF_AX25, GFP_ATOMIC, &ax25_proto, kern);
        if (sk == NULL)
                return -ENOMEM;

        ax25 = ax25_sk(sk)->cb = ax25_create_cb();
        if (!ax25) {
                sk_free(sk);
                return -ENOMEM;
        }

        sock_init_data(sock, sk);

        sk->sk_destruct = ax25_free_sock;
        sock->ops    = &ax25_proto_ops;
        sk->sk_protocol = protocol;

        ax25->sk    = sk;

        return 0;
}

struct sock *ax25_make_new(struct sock *osk, struct ax25_dev *ax25_dev)
{
        struct sock *sk;
        ax25_cb *ax25, *oax25;

        sk = sk_alloc(sock_net(osk), PF_AX25, GFP_ATOMIC, osk->sk_prot, 0);
        if (sk == NULL)
                return NULL;

        if ((ax25 = ax25_create_cb()) == NULL) {
                sk_free(sk);
                return NULL;
        }

        switch (osk->sk_type) {
        case SOCK_DGRAM:
                break;
        case SOCK_SEQPACKET:
                break;
        default:
                sk_free(sk);
                ax25_cb_put(ax25);
                return NULL;
        }

        sock_init_data(NULL, sk);

        sk->sk_type     = osk->sk_type;
        sk->sk_priority = READ_ONCE(osk->sk_priority);
        sk->sk_protocol = osk->sk_protocol;
        sk->sk_rcvbuf   = osk->sk_rcvbuf;
        sk->sk_sndbuf   = osk->sk_sndbuf;
        sk->sk_state    = TCP_ESTABLISHED;
        sock_copy_flags(sk, osk);

        oax25 = sk_to_ax25(osk);

        ax25->modulus = oax25->modulus;
        ax25->backoff = oax25->backoff;
        ax25->pidincl = oax25->pidincl;
        ax25->iamdigi = oax25->iamdigi;
        ax25->rtt     = oax25->rtt;
        ax25->t1      = oax25->t1;
        ax25->t2      = oax25->t2;
        ax25->t3      = oax25->t3;
        ax25->n2      = oax25->n2;
        ax25->idle    = oax25->idle;
        ax25->paclen  = oax25->paclen;
        ax25->window  = oax25->window;

        ax25->ax25_dev    = ax25_dev;
        ax25->source_addr = oax25->source_addr;

        if (oax25->digipeat != NULL) {
                ax25->digipeat = kmemdup(oax25->digipeat, sizeof(ax25_digi),
                                         GFP_ATOMIC);
                if (ax25->digipeat == NULL) {
                        sk_free(sk);
                        ax25_cb_put(ax25);
                        return NULL;
                }
        }

        ax25_sk(sk)->cb = ax25;
        sk->sk_destruct = ax25_free_sock;
        ax25->sk    = sk;

        return sk;
}

static int ax25_release(struct socket *sock)
{
        struct sock *sk = sock->sk;
        ax25_cb *ax25;
        ax25_dev *ax25_dev;

        if (sk == NULL)
                return 0;

        sock_hold(sk);
        lock_sock(sk);
        sock_orphan(sk);
        ax25 = sk_to_ax25(sk);
        ax25_dev = ax25->ax25_dev;

        if (sk->sk_type == SOCK_SEQPACKET) {
                switch (ax25->state) {
                case AX25_STATE_0:
                        if (!sock_flag(ax25->sk, SOCK_DEAD)) {
                                release_sock(sk);
                                ax25_disconnect(ax25, 0);
                                lock_sock(sk);
                        }
                        ax25_destroy_socket(ax25);
                        break;

                case AX25_STATE_1:
                case AX25_STATE_2:
                        ax25_send_control(ax25, AX25_DISC, AX25_POLLON, AX25_COMMAND);
                        release_sock(sk);
                        ax25_disconnect(ax25, 0);
                        lock_sock(sk);
                        if (!sock_flag(ax25->sk, SOCK_DESTROY))
                                ax25_destroy_socket(ax25);
                        break;

                case AX25_STATE_3:
                case AX25_STATE_4:
                        ax25_clear_queues(ax25);
                        ax25->n2count = 0;

                        switch (ax25->ax25_dev->values[AX25_VALUES_PROTOCOL]) {
                        case AX25_PROTO_STD_SIMPLEX:
                        case AX25_PROTO_STD_DUPLEX:
                                ax25_send_control(ax25,
                                                  AX25_DISC,
                                                  AX25_POLLON,
                                                  AX25_COMMAND);
                                ax25_stop_t2timer(ax25);
                                ax25_stop_t3timer(ax25);
                                ax25_stop_idletimer(ax25);
                                break;
#ifdef CONFIG_AX25_DAMA_SLAVE
                        case AX25_PROTO_DAMA_SLAVE:
                                ax25_stop_t3timer(ax25);
                                ax25_stop_idletimer(ax25);
                                break;
#endif
                        }
                        ax25_calculate_t1(ax25);
                        ax25_start_t1timer(ax25);
                        ax25->state = AX25_STATE_2;
                        sk->sk_state                = TCP_CLOSE;
                        sk->sk_shutdown            |= SEND_SHUTDOWN;
                        sk->sk_state_change(sk);
                        sock_set_flag(sk, SOCK_DESTROY);
                        break;

                default:
                        break;
                }
        } else {
                sk->sk_state     = TCP_CLOSE;
                sk->sk_shutdown |= SEND_SHUTDOWN;
                sk->sk_state_change(sk);
                ax25_destroy_socket(ax25);
        }
        if (ax25_dev) {
                if (!ax25_dev->device_up) {
                        timer_delete_sync(&ax25->timer);
                        timer_delete_sync(&ax25->t1timer);
                        timer_delete_sync(&ax25->t2timer);
                        timer_delete_sync(&ax25->t3timer);
                        timer_delete_sync(&ax25->idletimer);
                }
                netdev_put(ax25_dev->dev, &ax25->dev_tracker);
                ax25_dev_put(ax25_dev);
        }

        sock->sk   = NULL;
        release_sock(sk);
        sock_put(sk);

        return 0;
}

/*
 *        We support a funny extension here so you can (as root) give any callsign
 *        digipeated via a local address as source. This hack is obsolete now
 *        that we've implemented support for SO_BINDTODEVICE. It is however small
 *        and trivially backward compatible.
 */
static int ax25_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
{
        struct sock *sk = sock->sk;
        struct full_sockaddr_ax25 *addr = (struct full_sockaddr_ax25 *)uaddr;
        ax25_dev *ax25_dev = NULL;
        ax25_uid_assoc *user;
        ax25_address call;
        ax25_cb *ax25;
        int err = 0;

        if (addr_len != sizeof(struct sockaddr_ax25) &&
            addr_len != sizeof(struct full_sockaddr_ax25))
                /* support for old structure may go away some time
                 * ax25_bind(): uses old (6 digipeater) socket structure.
                 */
                if ((addr_len < sizeof(struct sockaddr_ax25) + sizeof(ax25_address) * 6) ||
                    (addr_len > sizeof(struct full_sockaddr_ax25)))
                        return -EINVAL;

        if (addr->fsa_ax25.sax25_family != AF_AX25)
                return -EINVAL;

        user = ax25_findbyuid(current_euid());
        if (user) {
                call = user->call;
                ax25_uid_put(user);
        } else {
                if (ax25_uid_policy && !capable(CAP_NET_ADMIN))
                        return -EACCES;

                call = addr->fsa_ax25.sax25_call;
        }

        lock_sock(sk);

        ax25 = sk_to_ax25(sk);
        if (!sock_flag(sk, SOCK_ZAPPED)) {
                err = -EINVAL;
                goto out;
        }

        ax25->source_addr = call;

        /*
         * User already set interface with SO_BINDTODEVICE
         */
        if (ax25->ax25_dev != NULL)
                goto done;

        if (addr_len > sizeof(struct sockaddr_ax25) && addr->fsa_ax25.sax25_ndigis == 1) {
                if (ax25cmp(&addr->fsa_digipeater[0], &null_ax25_address) != 0 &&
                    (ax25_dev = ax25_addr_ax25dev(&addr->fsa_digipeater[0])) == NULL) {
                        err = -EADDRNOTAVAIL;
                        goto out;
                }
        } else {
                if ((ax25_dev = ax25_addr_ax25dev(&addr->fsa_ax25.sax25_call)) == NULL) {
                        err = -EADDRNOTAVAIL;
                        goto out;
                }
        }

        if (ax25_dev) {
                ax25_fillin_cb(ax25, ax25_dev);
                netdev_hold(ax25_dev->dev, &ax25->dev_tracker, GFP_ATOMIC);
        }

done:
        ax25_cb_add(ax25);
        sock_reset_flag(sk, SOCK_ZAPPED);

out:
        release_sock(sk);

        return err;
}

/*
 *        FIXME: nonblock behaviour looks like it may have a bug.
 */
static int __must_check ax25_connect(struct socket *sock,
        struct sockaddr *uaddr, int addr_len, int flags)
{
        struct sock *sk = sock->sk;
        ax25_cb *ax25 = sk_to_ax25(sk), *ax25t;
        struct full_sockaddr_ax25 *fsa = (struct full_sockaddr_ax25 *)uaddr;
        ax25_digi *digi = NULL;
        int ct = 0, err = 0;

        /*
         * some sanity checks. code further down depends on this
         */

        if (addr_len == sizeof(struct sockaddr_ax25))
                /* support for this will go away in early 2.5.x
                 * ax25_connect(): uses obsolete socket structure
                 */
                ;
        else if (addr_len != sizeof(struct full_sockaddr_ax25))
                /* support for old structure may go away some time
                 * ax25_connect(): uses old (6 digipeater) socket structure.
                 */
                if ((addr_len < sizeof(struct sockaddr_ax25) + sizeof(ax25_address) * 6) ||
                    (addr_len > sizeof(struct full_sockaddr_ax25)))
                        return -EINVAL;


        if (fsa->fsa_ax25.sax25_family != AF_AX25)
                return -EINVAL;

        lock_sock(sk);

        /* deal with restarts */
        if (sock->state == SS_CONNECTING) {
                switch (sk->sk_state) {
                case TCP_SYN_SENT: /* still trying */
                        err = -EINPROGRESS;
                        goto out_release;

                case TCP_ESTABLISHED: /* connection established */
                        sock->state = SS_CONNECTED;
                        goto out_release;

                case TCP_CLOSE: /* connection refused */
                        sock->state = SS_UNCONNECTED;
                        err = -ECONNREFUSED;
                        goto out_release;
                }
        }

        if (sk->sk_state == TCP_ESTABLISHED && sk->sk_type == SOCK_SEQPACKET) {
                err = -EISCONN;        /* No reconnect on a seqpacket socket */
                goto out_release;
        }

        sk->sk_state   = TCP_CLOSE;
        sock->state = SS_UNCONNECTED;

        kfree(ax25->digipeat);
        ax25->digipeat = NULL;

        /*
         *        Handle digi-peaters to be used.
         */
        if (addr_len > sizeof(struct sockaddr_ax25) &&
            fsa->fsa_ax25.sax25_ndigis != 0) {
                /* Valid number of digipeaters ? */
                if (fsa->fsa_ax25.sax25_ndigis < 1 ||
                    fsa->fsa_ax25.sax25_ndigis > AX25_MAX_DIGIS ||
                    addr_len < sizeof(struct sockaddr_ax25) +
                    sizeof(ax25_address) * fsa->fsa_ax25.sax25_ndigis) {
                        err = -EINVAL;
                        goto out_release;
                }

                if ((digi = kmalloc(sizeof(ax25_digi), GFP_KERNEL)) == NULL) {
                        err = -ENOBUFS;
                        goto out_release;
                }

                digi->ndigi      = fsa->fsa_ax25.sax25_ndigis;
                digi->lastrepeat = -1;

                while (ct < fsa->fsa_ax25.sax25_ndigis) {
                        if ((fsa->fsa_digipeater[ct].ax25_call[6] &
                             AX25_HBIT) && ax25->iamdigi) {
                                digi->repeated[ct] = 1;
                                digi->lastrepeat   = ct;
                        } else {
                                digi->repeated[ct] = 0;
                        }
                        digi->calls[ct] = fsa->fsa_digipeater[ct];
                        ct++;
                }
        }

        /* Must bind first - autobinding does not work. */
        if (sock_flag(sk, SOCK_ZAPPED)) {
                kfree(digi);
                err = -EINVAL;
                goto out_release;
        }

        /* Check to see if the device has been filled in, error if it hasn't. */
        if (ax25->ax25_dev == NULL) {
                kfree(digi);
                err = -EHOSTUNREACH;
                goto out_release;
        }

        if (sk->sk_type == SOCK_SEQPACKET &&
            (ax25t=ax25_find_cb(&ax25->source_addr, &fsa->fsa_ax25.sax25_call, digi,
                         ax25->ax25_dev->dev))) {
                kfree(digi);
                err = -EADDRINUSE;                /* Already such a connection */
                ax25_cb_put(ax25t);
                goto out_release;
        }

        ax25->dest_addr = fsa->fsa_ax25.sax25_call;
        ax25->digipeat  = digi;

        /* First the easy one */
        if (sk->sk_type != SOCK_SEQPACKET) {
                sock->state = SS_CONNECTED;
                sk->sk_state   = TCP_ESTABLISHED;
                goto out_release;
        }

        /* Move to connecting socket, ax.25 lapb WAIT_UA.. */
        sock->state        = SS_CONNECTING;
        sk->sk_state          = TCP_SYN_SENT;

        switch (ax25->ax25_dev->values[AX25_VALUES_PROTOCOL]) {
        case AX25_PROTO_STD_SIMPLEX:
        case AX25_PROTO_STD_DUPLEX:
                ax25_std_establish_data_link(ax25);
                break;

#ifdef CONFIG_AX25_DAMA_SLAVE
        case AX25_PROTO_DAMA_SLAVE:
                ax25->modulus = AX25_MODULUS;
                ax25->window  = ax25->ax25_dev->values[AX25_VALUES_WINDOW];
                if (ax25->ax25_dev->dama.slave)
                        ax25_ds_establish_data_link(ax25);
                else
                        ax25_std_establish_data_link(ax25);
                break;
#endif
        }

        ax25->state = AX25_STATE_1;

        ax25_start_heartbeat(ax25);

        /* Now the loop */
        if (sk->sk_state != TCP_ESTABLISHED && (flags & O_NONBLOCK)) {
                err = -EINPROGRESS;
                goto out_release;
        }

        if (sk->sk_state == TCP_SYN_SENT) {
                DEFINE_WAIT(wait);

                for (;;) {
                        prepare_to_wait(sk_sleep(sk), &wait,
                                        TASK_INTERRUPTIBLE);
                        if (sk->sk_state != TCP_SYN_SENT)
                                break;
                        if (!signal_pending(current)) {
                                release_sock(sk);
                                schedule();
                                lock_sock(sk);
                                continue;
                        }
                        err = -ERESTARTSYS;
                        break;
                }
                finish_wait(sk_sleep(sk), &wait);

                if (err)
                        goto out_release;
        }

        if (sk->sk_state != TCP_ESTABLISHED) {
                /* Not in ABM, not in WAIT_UA -> failed */
                sock->state = SS_UNCONNECTED;
                err = sock_error(sk);        /* Always set at this point */
                goto out_release;
        }

        sock->state = SS_CONNECTED;

        err = 0;
out_release:
        release_sock(sk);

        return err;
}

static int ax25_accept(struct socket *sock, struct socket *newsock,
                       struct proto_accept_arg *arg)
{
        struct sk_buff *skb;
        struct sock *newsk;
        ax25_dev *ax25_dev;
        DEFINE_WAIT(wait);
        struct sock *sk;
        ax25_cb *ax25;
        int err = 0;

        if (sock->state != SS_UNCONNECTED)
                return -EINVAL;

        if ((sk = sock->sk) == NULL)
                return -EINVAL;

        lock_sock(sk);
        if (sk->sk_type != SOCK_SEQPACKET) {
                err = -EOPNOTSUPP;
                goto out;
        }

        if (sk->sk_state != TCP_LISTEN) {
                err = -EINVAL;
                goto out;
        }

        /*
         *        The read queue this time is holding sockets ready to use
         *        hooked into the SABM we saved
         */
        for (;;) {
                prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
                skb = skb_dequeue(&sk->sk_receive_queue);
                if (skb)
                        break;

                if (arg->flags & O_NONBLOCK) {
                        err = -EWOULDBLOCK;
                        break;
                }
                if (!signal_pending(current)) {
                        release_sock(sk);
                        schedule();
                        lock_sock(sk);
                        continue;
                }
                err = -ERESTARTSYS;
                break;
        }
        finish_wait(sk_sleep(sk), &wait);

        if (err)
                goto out;

        newsk                 = skb->sk;
        sock_graft(newsk, newsock);

        /* Now attach up the new socket */
        kfree_skb(skb);
        sk_acceptq_removed(sk);
        newsock->state = SS_CONNECTED;
        ax25 = sk_to_ax25(newsk);
        ax25_dev = ax25->ax25_dev;
        netdev_hold(ax25_dev->dev, &ax25->dev_tracker, GFP_ATOMIC);
        ax25_dev_hold(ax25_dev);

out:
        release_sock(sk);

        return err;
}

static int ax25_getname(struct socket *sock, struct sockaddr *uaddr,
        int peer)
{
        struct full_sockaddr_ax25 *fsa = (struct full_sockaddr_ax25 *)uaddr;
        struct sock *sk = sock->sk;
        unsigned char ndigi, i;
        ax25_cb *ax25;
        int err = 0;

        memset(fsa, 0, sizeof(*fsa));
        lock_sock(sk);
        ax25 = sk_to_ax25(sk);

        if (peer != 0) {
                if (sk->sk_state != TCP_ESTABLISHED) {
                        err = -ENOTCONN;
                        goto out;
                }

                fsa->fsa_ax25.sax25_family = AF_AX25;
                fsa->fsa_ax25.sax25_call   = ax25->dest_addr;

                if (ax25->digipeat != NULL) {
                        ndigi = ax25->digipeat->ndigi;
                        fsa->fsa_ax25.sax25_ndigis = ndigi;
                        for (i = 0; i < ndigi; i++)
                                fsa->fsa_digipeater[i] =
                                                ax25->digipeat->calls[i];
                }
        } else {
                fsa->fsa_ax25.sax25_family = AF_AX25;
                fsa->fsa_ax25.sax25_call   = ax25->source_addr;
                fsa->fsa_ax25.sax25_ndigis = 1;
                if (ax25->ax25_dev != NULL) {
                        memcpy(&fsa->fsa_digipeater[0],
                               ax25->ax25_dev->dev->dev_addr, AX25_ADDR_LEN);
                } else {
                        fsa->fsa_digipeater[0] = null_ax25_address;
                }
        }
        err = sizeof (struct full_sockaddr_ax25);

out:
        release_sock(sk);

        return err;
}

static int ax25_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
{
        DECLARE_SOCKADDR(struct sockaddr_ax25 *, usax, msg->msg_name);
        struct sock *sk = sock->sk;
        struct sockaddr_ax25 sax;
        struct sk_buff *skb;
        ax25_digi dtmp, *dp;
        ax25_cb *ax25;
        size_t size;
        int lv, err, addr_len = msg->msg_namelen;

        if (msg->msg_flags & ~(MSG_DONTWAIT|MSG_EOR|MSG_CMSG_COMPAT))
                return -EINVAL;

        lock_sock(sk);
        ax25 = sk_to_ax25(sk);

        if (sock_flag(sk, SOCK_ZAPPED)) {
                err = -EADDRNOTAVAIL;
                goto out;
        }

        if (sk->sk_shutdown & SEND_SHUTDOWN) {
                send_sig(SIGPIPE, current, 0);
                err = -EPIPE;
                goto out;
        }

        if (ax25->ax25_dev == NULL) {
                err = -ENETUNREACH;
                goto out;
        }

        if (len > ax25->ax25_dev->dev->mtu) {
                err = -EMSGSIZE;
                goto out;
        }

        if (usax != NULL) {
                if (usax->sax25_family != AF_AX25) {
                        err = -EINVAL;
                        goto out;
                }

                if (addr_len == sizeof(struct sockaddr_ax25))
                        /* ax25_sendmsg(): uses obsolete socket structure */
                        ;
                else if (addr_len != sizeof(struct full_sockaddr_ax25))
                        /* support for old structure may go away some time
                         * ax25_sendmsg(): uses old (6 digipeater)
                         * socket structure.
                         */
                        if ((addr_len < sizeof(struct sockaddr_ax25) + sizeof(ax25_address) * 6) ||
                            (addr_len > sizeof(struct full_sockaddr_ax25))) {
                                err = -EINVAL;
                                goto out;
                        }


                if (addr_len > sizeof(struct sockaddr_ax25) && usax->sax25_ndigis != 0) {
                        int ct           = 0;
                        struct full_sockaddr_ax25 *fsa = (struct full_sockaddr_ax25 *)usax;

                        /* Valid number of digipeaters ? */
                        if (usax->sax25_ndigis < 1 ||
                            usax->sax25_ndigis > AX25_MAX_DIGIS ||
                            addr_len < sizeof(struct sockaddr_ax25) +
                            sizeof(ax25_address) * usax->sax25_ndigis) {
                                err = -EINVAL;
                                goto out;
                        }

                        dtmp.ndigi      = usax->sax25_ndigis;

                        while (ct < usax->sax25_ndigis) {
                                dtmp.repeated[ct] = 0;
                                dtmp.calls[ct]    = fsa->fsa_digipeater[ct];
                                ct++;
                        }

                        dtmp.lastrepeat = 0;
                }

                sax = *usax;
                if (sk->sk_type == SOCK_SEQPACKET &&
                    ax25cmp(&ax25->dest_addr, &sax.sax25_call)) {
                        err = -EISCONN;
                        goto out;
                }
                if (usax->sax25_ndigis == 0)
                        dp = NULL;
                else
                        dp = &dtmp;
        } else {
                /*
                 *        FIXME: 1003.1g - if the socket is like this because
                 *        it has become closed (not started closed) and is VC
                 *        we ought to SIGPIPE, EPIPE
                 */
                if (sk->sk_state != TCP_ESTABLISHED) {
                        err = -ENOTCONN;
                        goto out;
                }
                sax.sax25_family = AF_AX25;
                sax.sax25_call   = ax25->dest_addr;
                dp = ax25->digipeat;
        }

        /* Build a packet */
        /* Assume the worst case */
        size = len + ax25->ax25_dev->dev->hard_header_len;

        skb = sock_alloc_send_skb(sk, size, msg->msg_flags&MSG_DONTWAIT, &err);
        if (skb == NULL)
                goto out;

        skb_reserve(skb, size - len);

        /* User data follows immediately after the AX.25 data */
        if (memcpy_from_msg(skb_put(skb, len), msg, len)) {
                err = -EFAULT;
                kfree_skb(skb);
                goto out;
        }

        skb_reset_network_header(skb);

        /* Add the PID if one is not supplied by the user in the skb */
        if (!ax25->pidincl)
                *(u8 *)skb_push(skb, 1) = sk->sk_protocol;

        if (sk->sk_type == SOCK_SEQPACKET) {
                /* Connected mode sockets go via the LAPB machine */
                if (sk->sk_state != TCP_ESTABLISHED) {
                        kfree_skb(skb);
                        err = -ENOTCONN;
                        goto out;
                }

                /* Shove it onto the queue and kick */
                ax25_output(ax25, ax25->paclen, skb);

                err = len;
                goto out;
        }

        skb_push(skb, 1 + ax25_addr_size(dp));

        /* Building AX.25 Header */

        /* Build an AX.25 header */
        lv = ax25_addr_build(skb->data, &ax25->source_addr, &sax.sax25_call,
                             dp, AX25_COMMAND, AX25_MODULUS);

        skb_set_transport_header(skb, lv);

        *skb_transport_header(skb) = AX25_UI;

        /* Datagram frames go straight out of the door as UI */
        ax25_queue_xmit(skb, ax25->ax25_dev->dev);

        err = len;

out:
        release_sock(sk);

        return err;
}

static int ax25_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
                        int flags)
{
        struct sock *sk = sock->sk;
        struct sk_buff *skb, *last;
        struct sk_buff_head *sk_queue;
        int copied;
        int err = 0;
        int off = 0;
        long timeo;

        lock_sock(sk);
        /*
         *         This works for seqpacket too. The receiver has ordered the
         *        queue for us! We do one quick check first though
         */
        if (sk->sk_type == SOCK_SEQPACKET && sk->sk_state != TCP_ESTABLISHED) {
                err =  -ENOTCONN;
                goto out;
        }

        /*  We need support for non-blocking reads. */
        sk_queue = &sk->sk_receive_queue;
        skb = __skb_try_recv_datagram(sk, sk_queue, flags, &off, &err, &last);
        /* If no packet is available, release_sock(sk) and try again. */
        if (!skb) {
                if (err != -EAGAIN)
                        goto out;
                release_sock(sk);
                timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
                while (timeo && !__skb_wait_for_more_packets(sk, sk_queue, &err,
                                                             &timeo, last)) {
                        skb = __skb_try_recv_datagram(sk, sk_queue, flags, &off,
                                                      &err, &last);
                        if (skb)
                                break;

                        if (err != -EAGAIN)
                                goto done;
                }
                if (!skb)
                        goto done;
                lock_sock(sk);
        }

        if (!sk_to_ax25(sk)->pidincl)
                skb_pull(skb, 1);                /* Remove PID */

        skb_reset_transport_header(skb);
        copied = skb->len;

        if (copied > size) {
                copied = size;
                msg->msg_flags |= MSG_TRUNC;
        }

        skb_copy_datagram_msg(skb, 0, msg, copied);

        if (msg->msg_name) {
                ax25_digi digi;
                ax25_address src;
                const unsigned char *mac = skb_mac_header(skb);
                DECLARE_SOCKADDR(struct sockaddr_ax25 *, sax, msg->msg_name);

                memset(sax, 0, sizeof(struct full_sockaddr_ax25));
                ax25_addr_parse(mac + 1, skb->data - mac - 1, &src, NULL,
                                &digi, NULL, NULL);
                sax->sax25_family = AF_AX25;
                /* We set this correctly, even though we may not let the
                   application know the digi calls further down (because it
                   did NOT ask to know them).  This could get political... **/
                sax->sax25_ndigis = digi.ndigi;
                sax->sax25_call   = src;

                if (sax->sax25_ndigis != 0) {
                        int ct;
                        struct full_sockaddr_ax25 *fsa = (struct full_sockaddr_ax25 *)sax;

                        for (ct = 0; ct < digi.ndigi; ct++)
                                fsa->fsa_digipeater[ct] = digi.calls[ct];
                }
                msg->msg_namelen = sizeof(struct full_sockaddr_ax25);
        }

        skb_free_datagram(sk, skb);
        err = copied;

out:
        release_sock(sk);

done:
        return err;
}

static int ax25_shutdown(struct socket *sk, int how)
{
        /* FIXME - generate DM and RNR states */
        return -EOPNOTSUPP;
}

static int ax25_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
{
        struct sock *sk = sock->sk;
        void __user *argp = (void __user *)arg;
        int res = 0;

        lock_sock(sk);
        switch (cmd) {
        case TIOCOUTQ: {
                long amount;

                amount = sk->sk_sndbuf - sk_wmem_alloc_get(sk);
                if (amount < 0)
                        amount = 0;
                res = put_user(amount, (int __user *)argp);
                break;
        }

        case TIOCINQ: {
                struct sk_buff *skb;
                long amount = 0L;
                /* These two are safe on a single CPU system as only user tasks fiddle here */
                if ((skb = skb_peek(&sk->sk_receive_queue)) != NULL)
                        amount = skb->len;
                res = put_user(amount, (int __user *) argp);
                break;
        }

        case SIOCAX25ADDUID:        /* Add a uid to the uid/call map table */
        case SIOCAX25DELUID:        /* Delete a uid from the uid/call map table */
        case SIOCAX25GETUID: {
                struct sockaddr_ax25 sax25;
                if (copy_from_user(&sax25, argp, sizeof(sax25))) {
                        res = -EFAULT;
                        break;
                }
                res = ax25_uid_ioctl(cmd, &sax25);
                break;
        }

        case SIOCAX25NOUID: {        /* Set the default policy (default/bar) */
                long amount;
                if (!capable(CAP_NET_ADMIN)) {
                        res = -EPERM;
                        break;
                }
                if (get_user(amount, (long __user *)argp)) {
                        res = -EFAULT;
                        break;
                }
                if (amount < 0 || amount > AX25_NOUID_BLOCK) {
                        res = -EINVAL;
                        break;
                }
                ax25_uid_policy = amount;
                res = 0;
                break;
        }

        case SIOCADDRT:
        case SIOCDELRT:
        case SIOCAX25OPTRT:
                if (!capable(CAP_NET_ADMIN)) {
                        res = -EPERM;
                        break;
                }
                res = ax25_rt_ioctl(cmd, argp);
                break;

        case SIOCAX25CTLCON:
                if (!capable(CAP_NET_ADMIN)) {
                        res = -EPERM;
                        break;
                }
                res = ax25_ctl_ioctl(cmd, argp);
                break;

        case SIOCAX25GETINFO:
        case SIOCAX25GETINFOOLD: {
                ax25_cb *ax25 = sk_to_ax25(sk);
                struct ax25_info_struct ax25_info;

                ax25_info.t1        = ax25->t1   / HZ;
                ax25_info.t2        = ax25->t2   / HZ;
                ax25_info.t3        = ax25->t3   / HZ;
                ax25_info.idle      = ax25->idle / (60 * HZ);
                ax25_info.n2        = ax25->n2;
                ax25_info.t1timer   = ax25_display_timer(&ax25->t1timer)   / HZ;
                ax25_info.t2timer   = ax25_display_timer(&ax25->t2timer)   / HZ;
                ax25_info.t3timer   = ax25_display_timer(&ax25->t3timer)   / HZ;
                ax25_info.idletimer = ax25_display_timer(&ax25->idletimer) / (60 * HZ);
                ax25_info.n2count   = ax25->n2count;
                ax25_info.state     = ax25->state;
                ax25_info.rcv_q     = sk_rmem_alloc_get(sk);
                ax25_info.snd_q     = sk_wmem_alloc_get(sk);
                ax25_info.vs        = ax25->vs;
                ax25_info.vr        = ax25->vr;
                ax25_info.va        = ax25->va;
                ax25_info.vs_max    = ax25->vs; /* reserved */
                ax25_info.paclen    = ax25->paclen;
                ax25_info.window    = ax25->window;

                /* old structure? */
                if (cmd == SIOCAX25GETINFOOLD) {
                        static int warned = 0;
                        if (!warned) {
                                printk(KERN_INFO "%s uses old SIOCAX25GETINFO\n",
                                        current->comm);
                                warned=1;
                        }

                        if (copy_to_user(argp, &ax25_info, sizeof(struct ax25_info_struct_deprecated))) {
                                res = -EFAULT;
                                break;
                        }
                } else {
                        if (copy_to_user(argp, &ax25_info, sizeof(struct ax25_info_struct))) {
                                res = -EINVAL;
                                break;
                        }
                }
                res = 0;
                break;
        }

        case SIOCAX25ADDFWD:
        case SIOCAX25DELFWD: {
                struct ax25_fwd_struct ax25_fwd;
                if (!capable(CAP_NET_ADMIN)) {
                        res = -EPERM;
                        break;
                }
                if (copy_from_user(&ax25_fwd, argp, sizeof(ax25_fwd))) {
                        res = -EFAULT;
                        break;
                }
                res = ax25_fwd_ioctl(cmd, &ax25_fwd);
                break;
        }

        case SIOCGIFADDR:
        case SIOCSIFADDR:
        case SIOCGIFDSTADDR:
        case SIOCSIFDSTADDR:
        case SIOCGIFBRDADDR:
        case SIOCSIFBRDADDR:
        case SIOCGIFNETMASK:
        case SIOCSIFNETMASK:
        case SIOCGIFMETRIC:
        case SIOCSIFMETRIC:
                res = -EINVAL;
                break;

        default:
                res = -ENOIOCTLCMD;
                break;
        }
        release_sock(sk);

        return res;
}

#ifdef CONFIG_PROC_FS

static void *ax25_info_start(struct seq_file *seq, loff_t *pos)
        __acquires(ax25_list_lock)
{
        spin_lock_bh(&ax25_list_lock);
        return seq_hlist_start(&ax25_list, *pos);
}

static void *ax25_info_next(struct seq_file *seq, void *v, loff_t *pos)
{
        return seq_hlist_next(v, &ax25_list, pos);
}

static void ax25_info_stop(struct seq_file *seq, void *v)
        __releases(ax25_list_lock)
{
        spin_unlock_bh(&ax25_list_lock);
}

static int ax25_info_show(struct seq_file *seq, void *v)
{
        ax25_cb *ax25 = hlist_entry(v, struct ax25_cb, ax25_node);
        char buf[11];
        int k;


        /*
         * New format:
         * magic dev src_addr dest_addr,digi1,digi2,.. st vs vr va t1 t1 t2 t2 t3 t3 idle idle n2 n2 rtt window paclen Snd-Q Rcv-Q inode
         */

        seq_printf(seq, "%p %s %s%s ",
                   ax25,
                   ax25->ax25_dev == NULL? "???" : ax25->ax25_dev->dev->name,
                   ax2asc(buf, &ax25->source_addr),
                   ax25->iamdigi? "*":"");
        seq_printf(seq, "%s", ax2asc(buf, &ax25->dest_addr));

        for (k=0; (ax25->digipeat != NULL) && (k < ax25->digipeat->ndigi); k++) {
                seq_printf(seq, ",%s%s",
                           ax2asc(buf, &ax25->digipeat->calls[k]),
                           ax25->digipeat->repeated[k]? "*":"");
        }

        seq_printf(seq, " %d %d %d %d %lu %lu %lu %lu %lu %lu %lu %lu %d %d %lu %d %d",
                   ax25->state,
                   ax25->vs, ax25->vr, ax25->va,
                   ax25_display_timer(&ax25->t1timer) / HZ, ax25->t1 / HZ,
                   ax25_display_timer(&ax25->t2timer) / HZ, ax25->t2 / HZ,
                   ax25_display_timer(&ax25->t3timer) / HZ, ax25->t3 / HZ,
                   ax25_display_timer(&ax25->idletimer) / (60 * HZ),
                   ax25->idle / (60 * HZ),
                   ax25->n2count, ax25->n2,
                   ax25->rtt / HZ,
                   ax25->window,
                   ax25->paclen);

        if (ax25->sk != NULL) {
                seq_printf(seq, " %d %d %lu\n",
                           sk_wmem_alloc_get(ax25->sk),
                           sk_rmem_alloc_get(ax25->sk),
                           sock_i_ino(ax25->sk));
        } else {
                seq_puts(seq, " * * *\n");
        }
        return 0;
}

static const struct seq_operations ax25_info_seqops = {
        .start = ax25_info_start,
        .next = ax25_info_next,
        .stop = ax25_info_stop,
        .show = ax25_info_show,
};
#endif

static const struct net_proto_family ax25_family_ops = {
        .family =        PF_AX25,
        .create =        ax25_create,
        .owner        =        THIS_MODULE,
};

static const struct proto_ops ax25_proto_ops = {
        .family                = PF_AX25,
        .owner                = THIS_MODULE,
        .release        = ax25_release,
        .bind                = ax25_bind,
        .connect        = ax25_connect,
        .socketpair        = sock_no_socketpair,
        .accept                = ax25_accept,
        .getname        = ax25_getname,
        .poll                = datagram_poll,
        .ioctl                = ax25_ioctl,
        .gettstamp        = sock_gettstamp,
        .listen                = ax25_listen,
        .shutdown        = ax25_shutdown,
        .setsockopt        = ax25_setsockopt,
        .getsockopt        = ax25_getsockopt,
        .sendmsg        = ax25_sendmsg,
        .recvmsg        = ax25_recvmsg,
        .mmap                = sock_no_mmap,
};

/*
 *        Called by socket.c on kernel start up
 */
static struct packet_type ax25_packet_type __read_mostly = {
        .type        =        cpu_to_be16(ETH_P_AX25),
        .func        =        ax25_kiss_rcv,
};

static struct notifier_block ax25_dev_notifier = {
        .notifier_call = ax25_device_event,
};

static int __init ax25_init(void)
{
        int rc = proto_register(&ax25_proto, 0);

        if (rc != 0)
                goto out;

        sock_register(&ax25_family_ops);
        dev_add_pack(&ax25_packet_type);
        register_netdevice_notifier(&ax25_dev_notifier);

        proc_create_seq("ax25_route", 0444, init_net.proc_net, &ax25_rt_seqops);
        proc_create_seq("ax25", 0444, init_net.proc_net, &ax25_info_seqops);
        proc_create_seq("ax25_calls", 0444, init_net.proc_net,
                        &ax25_uid_seqops);
out:
        return rc;
}
module_init(ax25_init);


MODULE_AUTHOR("Jonathan Naylor G4KLX <g4klx@g4klx.demon.co.uk>");
MODULE_DESCRIPTION("The amateur radio AX.25 link layer protocol");
MODULE_LICENSE("GPL");
MODULE_ALIAS_NETPROTO(PF_AX25);

static void __exit ax25_exit(void)
{
        remove_proc_entry("ax25_route", init_net.proc_net);
        remove_proc_entry("ax25", init_net.proc_net);
        remove_proc_entry("ax25_calls", init_net.proc_net);

        unregister_netdevice_notifier(&ax25_dev_notifier);

        dev_remove_pack(&ax25_packet_type);

        sock_unregister(PF_AX25);
        proto_unregister(&ax25_proto);

        ax25_rt_free();
        ax25_uid_free();
        ax25_dev_free();
}
module_exit(ax25_exit);














































































































   50 





































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
// SPDX-License-Identifier: GPL-2.0
/*
 * linux/ipc/shm.c
 * Copyright (C) 1992, 1993 Krishna Balasubramanian
 *         Many improvements/fixes by Bruno Haible.
 * Replaced `struct shm_desc' by `struct vm_area_struct', July 1994.
 * Fixed the shm swap deallocation (shm_unuse()), August 1998 Andrea Arcangeli.
 *
 * /proc/sysvipc/shm support (c) 1999 Dragos Acostachioaie <dragos@iname.com>
 * BIGMEM support, Andrea Arcangeli <andrea@suse.de>
 * SMP thread shm, Jean-Luc Boyard <jean-luc.boyard@siemens.fr>
 * HIGHMEM support, Ingo Molnar <mingo@redhat.com>
 * Make shmmax, shmall, shmmni sysctl'able, Christoph Rohland <cr@sap.com>
 * Shared /dev/zero support, Kanoj Sarcar <kanoj@sgi.com>
 * Move the mm functionality over to mm/shmem.c, Christoph Rohland <cr@sap.com>
 *
 * support for audit of ipc object properties and permission changes
 * Dustin Kirkland <dustin.kirkland@us.ibm.com>
 *
 * namespaces support
 * OpenVZ, SWsoft Inc.
 * Pavel Emelianov <xemul@openvz.org>
 *
 * Better ipc lock (kern_ipc_perm.lock) handling
 * Davidlohr Bueso <davidlohr.bueso@hp.com>, June 2013.
 */

#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/shm.h>
#include <uapi/linux/shm.h>
#include <linux/init.h>
#include <linux/file.h>
#include <linux/mman.h>
#include <linux/shmem_fs.h>
#include <linux/security.h>
#include <linux/syscalls.h>
#include <linux/audit.h>
#include <linux/capability.h>
#include <linux/ptrace.h>
#include <linux/seq_file.h>
#include <linux/rwsem.h>
#include <linux/nsproxy.h>
#include <linux/mount.h>
#include <linux/ipc_namespace.h>
#include <linux/rhashtable.h>

#include <linux/uaccess.h>

#include "util.h"

struct shmid_kernel /* private to the kernel */
{
        struct kern_ipc_perm        shm_perm;
        struct file                *shm_file;
        unsigned long                shm_nattch;
        unsigned long                shm_segsz;
        time64_t                shm_atim;
        time64_t                shm_dtim;
        time64_t                shm_ctim;
        struct pid                *shm_cprid;
        struct pid                *shm_lprid;
        struct ucounts                *mlock_ucounts;

        /*
         * The task created the shm object, for
         * task_lock(shp->shm_creator)
         */
        struct task_struct        *shm_creator;

        /*
         * List by creator. task_lock(->shm_creator) required for read/write.
         * If list_empty(), then the creator is dead already.
         */
        struct list_head        shm_clist;
        struct ipc_namespace        *ns;
} __randomize_layout;

/* shm_mode upper byte flags */
#define SHM_DEST        01000        /* segment will be destroyed on last detach */
#define SHM_LOCKED        02000   /* segment will not be swapped */

struct shm_file_data {
        int id;
        struct ipc_namespace *ns;
        struct file *file;
        const struct vm_operations_struct *vm_ops;
};

#define shm_file_data(file) (*((struct shm_file_data **)&(file)->private_data))

static const struct file_operations shm_file_operations;
static const struct vm_operations_struct shm_vm_ops;

#define shm_ids(ns)        ((ns)->ids[IPC_SHM_IDS])

#define shm_unlock(shp)                        \
        ipc_unlock(&(shp)->shm_perm)

static int newseg(struct ipc_namespace *, struct ipc_params *);
static void shm_open(struct vm_area_struct *vma);
static void shm_close(struct vm_area_struct *vma);
static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp);
#ifdef CONFIG_PROC_FS
static int sysvipc_shm_proc_show(struct seq_file *s, void *it);
#endif

void shm_init_ns(struct ipc_namespace *ns)
{
        ns->shm_ctlmax = SHMMAX;
        ns->shm_ctlall = SHMALL;
        ns->shm_ctlmni = SHMMNI;
        ns->shm_rmid_forced = 0;
        ns->shm_tot = 0;
        ipc_init_ids(&shm_ids(ns));
}

/*
 * Called with shm_ids.rwsem (writer) and the shp structure locked.
 * Only shm_ids.rwsem remains locked on exit.
 */
static void do_shm_rmid(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
{
        struct shmid_kernel *shp;

        shp = container_of(ipcp, struct shmid_kernel, shm_perm);
        WARN_ON(ns != shp->ns);

        if (shp->shm_nattch) {
                shp->shm_perm.mode |= SHM_DEST;
                /* Do not find it any more */
                ipc_set_key_private(&shm_ids(ns), &shp->shm_perm);
                shm_unlock(shp);
        } else
                shm_destroy(ns, shp);
}

#ifdef CONFIG_IPC_NS
void shm_exit_ns(struct ipc_namespace *ns)
{
        free_ipcs(ns, &shm_ids(ns), do_shm_rmid);
        idr_destroy(&ns->ids[IPC_SHM_IDS].ipcs_idr);
        rhashtable_destroy(&ns->ids[IPC_SHM_IDS].key_ht);
}
#endif

static int __init ipc_ns_init(void)
{
        shm_init_ns(&init_ipc_ns);
        return 0;
}

pure_initcall(ipc_ns_init);

void __init shm_init(void)
{
        ipc_init_proc_interface("sysvipc/shm",
#if BITS_PER_LONG <= 32
                                "       key      shmid perms       size  cpid  lpid nattch   uid   gid  cuid  cgid      atime      dtime      ctime        rss       swap\n",
#else
                                "       key      shmid perms                  size  cpid  lpid nattch   uid   gid  cuid  cgid      atime      dtime      ctime                   rss                  swap\n",
#endif
                                IPC_SHM_IDS, sysvipc_shm_proc_show);
}

static inline struct shmid_kernel *shm_obtain_object(struct ipc_namespace *ns, int id)
{
        struct kern_ipc_perm *ipcp = ipc_obtain_object_idr(&shm_ids(ns), id);

        if (IS_ERR(ipcp))
                return ERR_CAST(ipcp);

        return container_of(ipcp, struct shmid_kernel, shm_perm);
}

static inline struct shmid_kernel *shm_obtain_object_check(struct ipc_namespace *ns, int id)
{
        struct kern_ipc_perm *ipcp = ipc_obtain_object_check(&shm_ids(ns), id);

        if (IS_ERR(ipcp))
                return ERR_CAST(ipcp);

        return container_of(ipcp, struct shmid_kernel, shm_perm);
}

/*
 * shm_lock_(check_) routines are called in the paths where the rwsem
 * is not necessarily held.
 */
static inline struct shmid_kernel *shm_lock(struct ipc_namespace *ns, int id)
{
        struct kern_ipc_perm *ipcp;

        rcu_read_lock();
        ipcp = ipc_obtain_object_idr(&shm_ids(ns), id);
        if (IS_ERR(ipcp))
                goto err;

        ipc_lock_object(ipcp);
        /*
         * ipc_rmid() may have already freed the ID while ipc_lock_object()
         * was spinning: here verify that the structure is still valid.
         * Upon races with RMID, return -EIDRM, thus indicating that
         * the ID points to a removed identifier.
         */
        if (ipc_valid_object(ipcp)) {
                /* return a locked ipc object upon success */
                return container_of(ipcp, struct shmid_kernel, shm_perm);
        }

        ipc_unlock_object(ipcp);
        ipcp = ERR_PTR(-EIDRM);
err:
        rcu_read_unlock();
        /*
         * Callers of shm_lock() must validate the status of the returned ipc
         * object pointer and error out as appropriate.
         */
        return ERR_CAST(ipcp);
}

static inline void shm_lock_by_ptr(struct shmid_kernel *ipcp)
{
        rcu_read_lock();
        ipc_lock_object(&ipcp->shm_perm);
}

static void shm_rcu_free(struct rcu_head *head)
{
        struct kern_ipc_perm *ptr = container_of(head, struct kern_ipc_perm,
                                                        rcu);
        struct shmid_kernel *shp = container_of(ptr, struct shmid_kernel,
                                                        shm_perm);
        security_shm_free(&shp->shm_perm);
        kfree(shp);
}

/*
 * It has to be called with shp locked.
 * It must be called before ipc_rmid()
 */
static inline void shm_clist_rm(struct shmid_kernel *shp)
{
        struct task_struct *creator;

        /* ensure that shm_creator does not disappear */
        rcu_read_lock();

        /*
         * A concurrent exit_shm may do a list_del_init() as well.
         * Just do nothing if exit_shm already did the work
         */
        if (!list_empty(&shp->shm_clist)) {
                /*
                 * shp->shm_creator is guaranteed to be valid *only*
                 * if shp->shm_clist is not empty.
                 */
                creator = shp->shm_creator;

                task_lock(creator);
                /*
                 * list_del_init() is a nop if the entry was already removed
                 * from the list.
                 */
                list_del_init(&shp->shm_clist);
                task_unlock(creator);
        }
        rcu_read_unlock();
}

static inline void shm_rmid(struct shmid_kernel *s)
{
        shm_clist_rm(s);
        ipc_rmid(&shm_ids(s->ns), &s->shm_perm);
}


static int __shm_open(struct shm_file_data *sfd)
{
        struct shmid_kernel *shp;

        shp = shm_lock(sfd->ns, sfd->id);

        if (IS_ERR(shp))
                return PTR_ERR(shp);

        if (shp->shm_file != sfd->file) {
                /* ID was reused */
                shm_unlock(shp);
                return -EINVAL;
        }

        shp->shm_atim = ktime_get_real_seconds();
        ipc_update_pid(&shp->shm_lprid, task_tgid(current));
        shp->shm_nattch++;
        shm_unlock(shp);
        return 0;
}

/* This is called by fork, once for every shm attach. */
static void shm_open(struct vm_area_struct *vma)
{
        struct file *file = vma->vm_file;
        struct shm_file_data *sfd = shm_file_data(file);
        int err;

        /* Always call underlying open if present */
        if (sfd->vm_ops->open)
                sfd->vm_ops->open(vma);

        err = __shm_open(sfd);
        /*
         * We raced in the idr lookup or with shm_destroy().
         * Either way, the ID is busted.
         */
        WARN_ON_ONCE(err);
}

/*
 * shm_destroy - free the struct shmid_kernel
 *
 * @ns: namespace
 * @shp: struct to free
 *
 * It has to be called with shp and shm_ids.rwsem (writer) locked,
 * but returns with shp unlocked and freed.
 */
static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
{
        struct file *shm_file;

        shm_file = shp->shm_file;
        shp->shm_file = NULL;
        ns->shm_tot -= (shp->shm_segsz + PAGE_SIZE - 1) >> PAGE_SHIFT;
        shm_rmid(shp);
        shm_unlock(shp);
        if (!is_file_hugepages(shm_file))
                shmem_lock(shm_file, 0, shp->mlock_ucounts);
        fput(shm_file);
        ipc_update_pid(&shp->shm_cprid, NULL);
        ipc_update_pid(&shp->shm_lprid, NULL);
        ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);
}

/*
 * shm_may_destroy - identifies whether shm segment should be destroyed now
 *
 * Returns true if and only if there are no active users of the segment and
 * one of the following is true:
 *
 * 1) shmctl(id, IPC_RMID, NULL) was called for this shp
 *
 * 2) sysctl kernel.shm_rmid_forced is set to 1.
 */
static bool shm_may_destroy(struct shmid_kernel *shp)
{
        return (shp->shm_nattch == 0) &&
               (shp->ns->shm_rmid_forced ||
                (shp->shm_perm.mode & SHM_DEST));
}

/*
 * remove the attach descriptor vma.
 * free memory for segment if it is marked destroyed.
 * The descriptor has already been removed from the current->mm->mmap list
 * and will later be kfree()d.
 */
static void __shm_close(struct shm_file_data *sfd)
{
        struct shmid_kernel *shp;
        struct ipc_namespace *ns = sfd->ns;

        down_write(&shm_ids(ns).rwsem);
        /* remove from the list of attaches of the shm segment */
        shp = shm_lock(ns, sfd->id);

        /*
         * We raced in the idr lookup or with shm_destroy().
         * Either way, the ID is busted.
         */
        if (WARN_ON_ONCE(IS_ERR(shp)))
                goto done; /* no-op */

        ipc_update_pid(&shp->shm_lprid, task_tgid(current));
        shp->shm_dtim = ktime_get_real_seconds();
        shp->shm_nattch--;
        if (shm_may_destroy(shp))
                shm_destroy(ns, shp);
        else
                shm_unlock(shp);
done:
        up_write(&shm_ids(ns).rwsem);
}

static void shm_close(struct vm_area_struct *vma)
{
        struct file *file = vma->vm_file;
        struct shm_file_data *sfd = shm_file_data(file);

        /* Always call underlying close if present */
        if (sfd->vm_ops->close)
                sfd->vm_ops->close(vma);

        __shm_close(sfd);
}

/* Called with ns->shm_ids(ns).rwsem locked */
static int shm_try_destroy_orphaned(int id, void *p, void *data)
{
        struct ipc_namespace *ns = data;
        struct kern_ipc_perm *ipcp = p;
        struct shmid_kernel *shp = container_of(ipcp, struct shmid_kernel, shm_perm);

        /*
         * We want to destroy segments without users and with already
         * exit'ed originating process.
         *
         * As shp->* are changed under rwsem, it's safe to skip shp locking.
         */
        if (!list_empty(&shp->shm_clist))
                return 0;

        if (shm_may_destroy(shp)) {
                shm_lock_by_ptr(shp);
                shm_destroy(ns, shp);
        }
        return 0;
}

void shm_destroy_orphaned(struct ipc_namespace *ns)
{
        down_write(&shm_ids(ns).rwsem);
        if (shm_ids(ns).in_use)
                idr_for_each(&shm_ids(ns).ipcs_idr, &shm_try_destroy_orphaned, ns);
        up_write(&shm_ids(ns).rwsem);
}

/* Locking assumes this will only be called with task == current */
void exit_shm(struct task_struct *task)
{
        for (;;) {
                struct shmid_kernel *shp;
                struct ipc_namespace *ns;

                task_lock(task);

                if (list_empty(&task->sysvshm.shm_clist)) {
                        task_unlock(task);
                        break;
                }

                shp = list_first_entry(&task->sysvshm.shm_clist, struct shmid_kernel,
                                shm_clist);

                /*
                 * 1) Get pointer to the ipc namespace. It is worth to say
                 * that this pointer is guaranteed to be valid because
                 * shp lifetime is always shorter than namespace lifetime
                 * in which shp lives.
                 * We taken task_lock it means that shp won't be freed.
                 */
                ns = shp->ns;

                /*
                 * 2) If kernel.shm_rmid_forced is not set then only keep track of
                 * which shmids are orphaned, so that a later set of the sysctl
                 * can clean them up.
                 */
                if (!ns->shm_rmid_forced)
                        goto unlink_continue;

                /*
                 * 3) get a reference to the namespace.
                 *    The refcount could be already 0. If it is 0, then
                 *    the shm objects will be free by free_ipc_work().
                 */
                ns = get_ipc_ns_not_zero(ns);
                if (!ns) {
unlink_continue:
                        list_del_init(&shp->shm_clist);
                        task_unlock(task);
                        continue;
                }

                /*
                 * 4) get a reference to shp.
                 *   This cannot fail: shm_clist_rm() is called before
                 *   ipc_rmid(), thus the refcount cannot be 0.
                 */
                WARN_ON(!ipc_rcu_getref(&shp->shm_perm));

                /*
                 * 5) unlink the shm segment from the list of segments
                 *    created by current.
                 *    This must be done last. After unlinking,
                 *    only the refcounts obtained above prevent IPC_RMID
                 *    from destroying the segment or the namespace.
                 */
                list_del_init(&shp->shm_clist);

                task_unlock(task);

                /*
                 * 6) we have all references
                 *    Thus lock & if needed destroy shp.
                 */
                down_write(&shm_ids(ns).rwsem);
                shm_lock_by_ptr(shp);
                /*
                 * rcu_read_lock was implicitly taken in shm_lock_by_ptr, it's
                 * safe to call ipc_rcu_putref here
                 */
                ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);

                if (ipc_valid_object(&shp->shm_perm)) {
                        if (shm_may_destroy(shp))
                                shm_destroy(ns, shp);
                        else
                                shm_unlock(shp);
                } else {
                        /*
                         * Someone else deleted the shp from namespace
                         * idr/kht while we have waited.
                         * Just unlock and continue.
                         */
                        shm_unlock(shp);
                }

                up_write(&shm_ids(ns).rwsem);
                put_ipc_ns(ns); /* paired with get_ipc_ns_not_zero */
        }
}

static vm_fault_t shm_fault(struct vm_fault *vmf)
{
        struct file *file = vmf->vma->vm_file;
        struct shm_file_data *sfd = shm_file_data(file);

        return sfd->vm_ops->fault(vmf);
}

static int shm_may_split(struct vm_area_struct *vma, unsigned long addr)
{
        struct file *file = vma->vm_file;
        struct shm_file_data *sfd = shm_file_data(file);

        if (sfd->vm_ops->may_split)
                return sfd->vm_ops->may_split(vma, addr);

        return 0;
}

static unsigned long shm_pagesize(struct vm_area_struct *vma)
{
        struct file *file = vma->vm_file;
        struct shm_file_data *sfd = shm_file_data(file);

        if (sfd->vm_ops->pagesize)
                return sfd->vm_ops->pagesize(vma);

        return PAGE_SIZE;
}

#ifdef CONFIG_NUMA
static int shm_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
{
        struct shm_file_data *sfd = shm_file_data(vma->vm_file);
        int err = 0;

        if (sfd->vm_ops->set_policy)
                err = sfd->vm_ops->set_policy(vma, mpol);
        return err;
}

static struct mempolicy *shm_get_policy(struct vm_area_struct *vma,
                                        unsigned long addr, pgoff_t *ilx)
{
        struct shm_file_data *sfd = shm_file_data(vma->vm_file);
        struct mempolicy *mpol = vma->vm_policy;

        if (sfd->vm_ops->get_policy)
                mpol = sfd->vm_ops->get_policy(vma, addr, ilx);
        return mpol;
}
#endif

static int shm_mmap(struct file *file, struct vm_area_struct *vma)
{
        struct shm_file_data *sfd = shm_file_data(file);
        int ret;

        /*
         * In case of remap_file_pages() emulation, the file can represent an
         * IPC ID that was removed, and possibly even reused by another shm
         * segment already.  Propagate this case as an error to caller.
         */
        ret = __shm_open(sfd);
        if (ret)
                return ret;

        ret = call_mmap(sfd->file, vma);
        if (ret) {
                __shm_close(sfd);
                return ret;
        }
        sfd->vm_ops = vma->vm_ops;
#ifdef CONFIG_MMU
        WARN_ON(!sfd->vm_ops->fault);
#endif
        vma->vm_ops = &shm_vm_ops;
        return 0;
}

static int shm_release(struct inode *ino, struct file *file)
{
        struct shm_file_data *sfd = shm_file_data(file);

        put_ipc_ns(sfd->ns);
        fput(sfd->file);
        shm_file_data(file) = NULL;
        kfree(sfd);
        return 0;
}

static int shm_fsync(struct file *file, loff_t start, loff_t end, int datasync)
{
        struct shm_file_data *sfd = shm_file_data(file);

        if (!sfd->file->f_op->fsync)
                return -EINVAL;
        return sfd->file->f_op->fsync(sfd->file, start, end, datasync);
}

static long shm_fallocate(struct file *file, int mode, loff_t offset,
                          loff_t len)
{
        struct shm_file_data *sfd = shm_file_data(file);

        if (!sfd->file->f_op->fallocate)
                return -EOPNOTSUPP;
        return sfd->file->f_op->fallocate(file, mode, offset, len);
}

static unsigned long shm_get_unmapped_area(struct file *file,
        unsigned long addr, unsigned long len, unsigned long pgoff,
        unsigned long flags)
{
        struct shm_file_data *sfd = shm_file_data(file);

        return sfd->file->f_op->get_unmapped_area(sfd->file, addr, len,
                                                pgoff, flags);
}

static const struct file_operations shm_file_operations = {
        .mmap                = shm_mmap,
        .fsync                = shm_fsync,
        .release        = shm_release,
        .get_unmapped_area        = shm_get_unmapped_area,
        .llseek                = noop_llseek,
        .fallocate        = shm_fallocate,
};

/*
 * shm_file_operations_huge is now identical to shm_file_operations
 * except for fop_flags
 */
static const struct file_operations shm_file_operations_huge = {
        .mmap                = shm_mmap,
        .fsync                = shm_fsync,
        .release        = shm_release,
        .get_unmapped_area        = shm_get_unmapped_area,
        .llseek                = noop_llseek,
        .fallocate        = shm_fallocate,
        .fop_flags        = FOP_HUGE_PAGES,
};

static const struct vm_operations_struct shm_vm_ops = {
        .open        = shm_open,        /* callback for a new vm-area open */
        .close        = shm_close,        /* callback for when the vm-area is released */
        .fault        = shm_fault,
        .may_split = shm_may_split,
        .pagesize = shm_pagesize,
#if defined(CONFIG_NUMA)
        .set_policy = shm_set_policy,
        .get_policy = shm_get_policy,
#endif
};

/**
 * newseg - Create a new shared memory segment
 * @ns: namespace
 * @params: ptr to the structure that contains key, size and shmflg
 *
 * Called with shm_ids.rwsem held as a writer.
 */
static int newseg(struct ipc_namespace *ns, struct ipc_params *params)
{
        key_t key = params->key;
        int shmflg = params->flg;
        size_t size = params->u.size;
        int error;
        struct shmid_kernel *shp;
        size_t numpages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
        struct file *file;
        char name[13];
        vm_flags_t acctflag = 0;

        if (size < SHMMIN || size > ns->shm_ctlmax)
                return -EINVAL;

        if (numpages << PAGE_SHIFT < size)
                return -ENOSPC;

        if (ns->shm_tot + numpages < ns->shm_tot ||
                        ns->shm_tot + numpages > ns->shm_ctlall)
                return -ENOSPC;

        shp = kmalloc(sizeof(*shp), GFP_KERNEL_ACCOUNT);
        if (unlikely(!shp))
                return -ENOMEM;

        shp->shm_perm.key = key;
        shp->shm_perm.mode = (shmflg & S_IRWXUGO);
        shp->mlock_ucounts = NULL;

        shp->shm_perm.security = NULL;
        error = security_shm_alloc(&shp->shm_perm);
        if (error) {
                kfree(shp);
                return error;
        }

        sprintf(name, "SYSV%08x", key);
        if (shmflg & SHM_HUGETLB) {
                struct hstate *hs;
                size_t hugesize;

                hs = hstate_sizelog((shmflg >> SHM_HUGE_SHIFT) & SHM_HUGE_MASK);
                if (!hs) {
                        error = -EINVAL;
                        goto no_file;
                }
                hugesize = ALIGN(size, huge_page_size(hs));

                /* hugetlb_file_setup applies strict accounting */
                if (shmflg & SHM_NORESERVE)
                        acctflag = VM_NORESERVE;
                file = hugetlb_file_setup(name, hugesize, acctflag,
                                HUGETLB_SHMFS_INODE, (shmflg >> SHM_HUGE_SHIFT) & SHM_HUGE_MASK);
        } else {
                /*
                 * Do not allow no accounting for OVERCOMMIT_NEVER, even
                 * if it's asked for.
                 */
                if  ((shmflg & SHM_NORESERVE) &&
                                sysctl_overcommit_memory != OVERCOMMIT_NEVER)
                        acctflag = VM_NORESERVE;
                file = shmem_kernel_file_setup(name, size, acctflag);
        }
        error = PTR_ERR(file);
        if (IS_ERR(file))
                goto no_file;

        shp->shm_cprid = get_pid(task_tgid(current));
        shp->shm_lprid = NULL;
        shp->shm_atim = shp->shm_dtim = 0;
        shp->shm_ctim = ktime_get_real_seconds();
        shp->shm_segsz = size;
        shp->shm_nattch = 0;
        shp->shm_file = file;
        shp->shm_creator = current;

        /* ipc_addid() locks shp upon success. */
        error = ipc_addid(&shm_ids(ns), &shp->shm_perm, ns->shm_ctlmni);
        if (error < 0)
                goto no_id;

        shp->ns = ns;

        task_lock(current);
        list_add(&shp->shm_clist, &current->sysvshm.shm_clist);
        task_unlock(current);

        /*
         * shmid gets reported as "inode#" in /proc/pid/maps.
         * proc-ps tools use this. Changing this will break them.
         */
        file_inode(file)->i_ino = shp->shm_perm.id;

        ns->shm_tot += numpages;
        error = shp->shm_perm.id;

        ipc_unlock_object(&shp->shm_perm);
        rcu_read_unlock();
        return error;

no_id:
        ipc_update_pid(&shp->shm_cprid, NULL);
        ipc_update_pid(&shp->shm_lprid, NULL);
        fput(file);
        ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);
        return error;
no_file:
        call_rcu(&shp->shm_perm.rcu, shm_rcu_free);
        return error;
}

/*
 * Called with shm_ids.rwsem and ipcp locked.
 */
static int shm_more_checks(struct kern_ipc_perm *ipcp, struct ipc_params *params)
{
        struct shmid_kernel *shp;

        shp = container_of(ipcp, struct shmid_kernel, shm_perm);
        if (shp->shm_segsz < params->u.size)
                return -EINVAL;

        return 0;
}

long ksys_shmget(key_t key, size_t size, int shmflg)
{
        struct ipc_namespace *ns;
        static const struct ipc_ops shm_ops = {
                .getnew = newseg,
                .associate = security_shm_associate,
                .more_checks = shm_more_checks,
        };
        struct ipc_params shm_params;

        ns = current->nsproxy->ipc_ns;

        shm_params.key = key;
        shm_params.flg = shmflg;
        shm_params.u.size = size;

        return ipcget(ns, &shm_ids(ns), &shm_ops, &shm_params);
}

SYSCALL_DEFINE3(shmget, key_t, key, size_t, size, int, shmflg)
{
        return ksys_shmget(key, size, shmflg);
}

static inline unsigned long copy_shmid_to_user(void __user *buf, struct shmid64_ds *in, int version)
{
        switch (version) {
        case IPC_64:
                return copy_to_user(buf, in, sizeof(*in));
        case IPC_OLD:
            {
                struct shmid_ds out;

                memset(&out, 0, sizeof(out));
                ipc64_perm_to_ipc_perm(&in->shm_perm, &out.shm_perm);
                out.shm_segsz        = in->shm_segsz;
                out.shm_atime        = in->shm_atime;
                out.shm_dtime        = in->shm_dtime;
                out.shm_ctime        = in->shm_ctime;
                out.shm_cpid        = in->shm_cpid;
                out.shm_lpid        = in->shm_lpid;
                out.shm_nattch        = in->shm_nattch;

                return copy_to_user(buf, &out, sizeof(out));
            }
        default:
                return -EINVAL;
        }
}

static inline unsigned long
copy_shmid_from_user(struct shmid64_ds *out, void __user *buf, int version)
{
        switch (version) {
        case IPC_64:
                if (copy_from_user(out, buf, sizeof(*out)))
                        return -EFAULT;
                return 0;
        case IPC_OLD:
            {
                struct shmid_ds tbuf_old;

                if (copy_from_user(&tbuf_old, buf, sizeof(tbuf_old)))
                        return -EFAULT;

                out->shm_perm.uid        = tbuf_old.shm_perm.uid;
                out->shm_perm.gid        = tbuf_old.shm_perm.gid;
                out->shm_perm.mode        = tbuf_old.shm_perm.mode;

                return 0;
            }
        default:
                return -EINVAL;
        }
}

static inline unsigned long copy_shminfo_to_user(void __user *buf, struct shminfo64 *in, int version)
{
        switch (version) {
        case IPC_64:
                return copy_to_user(buf, in, sizeof(*in));
        case IPC_OLD:
            {
                struct shminfo out;

                if (in->shmmax > INT_MAX)
                        out.shmmax = INT_MAX;
                else
                        out.shmmax = (int)in->shmmax;

                out.shmmin        = in->shmmin;
                out.shmmni        = in->shmmni;
                out.shmseg        = in->shmseg;
                out.shmall        = in->shmall;

                return copy_to_user(buf, &out, sizeof(out));
            }
        default:
                return -EINVAL;
        }
}

/*
 * Calculate and add used RSS and swap pages of a shm.
 * Called with shm_ids.rwsem held as a reader
 */
static void shm_add_rss_swap(struct shmid_kernel *shp,
        unsigned long *rss_add, unsigned long *swp_add)
{
        struct inode *inode;

        inode = file_inode(shp->shm_file);

        if (is_file_hugepages(shp->shm_file)) {
                struct address_space *mapping = inode->i_mapping;
                struct hstate *h = hstate_file(shp->shm_file);
                *rss_add += pages_per_huge_page(h) * mapping->nrpages;
        } else {
#ifdef CONFIG_SHMEM
                struct shmem_inode_info *info = SHMEM_I(inode);

                spin_lock_irq(&info->lock);
                *rss_add += inode->i_mapping->nrpages;
                *swp_add += info->swapped;
                spin_unlock_irq(&info->lock);
#else
                *rss_add += inode->i_mapping->nrpages;
#endif
        }
}

/*
 * Called with shm_ids.rwsem held as a reader
 */
static void shm_get_stat(struct ipc_namespace *ns, unsigned long *rss,
                unsigned long *swp)
{
        int next_id;
        int total, in_use;

        *rss = 0;
        *swp = 0;

        in_use = shm_ids(ns).in_use;

        for (total = 0, next_id = 0; total < in_use; next_id++) {
                struct kern_ipc_perm *ipc;
                struct shmid_kernel *shp;

                ipc = idr_find(&shm_ids(ns).ipcs_idr, next_id);
                if (ipc == NULL)
                        continue;
                shp = container_of(ipc, struct shmid_kernel, shm_perm);

                shm_add_rss_swap(shp, rss, swp);

                total++;
        }
}

/*
 * This function handles some shmctl commands which require the rwsem
 * to be held in write mode.
 * NOTE: no locks must be held, the rwsem is taken inside this function.
 */
static int shmctl_down(struct ipc_namespace *ns, int shmid, int cmd,
                       struct shmid64_ds *shmid64)
{
        struct kern_ipc_perm *ipcp;
        struct shmid_kernel *shp;
        int err;

        down_write(&shm_ids(ns).rwsem);
        rcu_read_lock();

        ipcp = ipcctl_obtain_check(ns, &shm_ids(ns), shmid, cmd,
                                      &shmid64->shm_perm, 0);
        if (IS_ERR(ipcp)) {
                err = PTR_ERR(ipcp);
                goto out_unlock1;
        }

        shp = container_of(ipcp, struct shmid_kernel, shm_perm);

        err = security_shm_shmctl(&shp->shm_perm, cmd);
        if (err)
                goto out_unlock1;

        switch (cmd) {
        case IPC_RMID:
                ipc_lock_object(&shp->shm_perm);
                /* do_shm_rmid unlocks the ipc object and rcu */
                do_shm_rmid(ns, ipcp);
                goto out_up;
        case IPC_SET:
                ipc_lock_object(&shp->shm_perm);
                err = ipc_update_perm(&shmid64->shm_perm, ipcp);
                if (err)
                        goto out_unlock0;
                shp->shm_ctim = ktime_get_real_seconds();
                break;
        default:
                err = -EINVAL;
                goto out_unlock1;
        }

out_unlock0:
        ipc_unlock_object(&shp->shm_perm);
out_unlock1:
        rcu_read_unlock();
out_up:
        up_write(&shm_ids(ns).rwsem);
        return err;
}

static int shmctl_ipc_info(struct ipc_namespace *ns,
                           struct shminfo64 *shminfo)
{
        int err = security_shm_shmctl(NULL, IPC_INFO);
        if (!err) {
                memset(shminfo, 0, sizeof(*shminfo));
                shminfo->shmmni = shminfo->shmseg = ns->shm_ctlmni;
                shminfo->shmmax = ns->shm_ctlmax;
                shminfo->shmall = ns->shm_ctlall;
                shminfo->shmmin = SHMMIN;
                down_read(&shm_ids(ns).rwsem);
                err = ipc_get_maxidx(&shm_ids(ns));
                up_read(&shm_ids(ns).rwsem);
                if (err < 0)
                        err = 0;
        }
        return err;
}

static int shmctl_shm_info(struct ipc_namespace *ns,
                           struct shm_info *shm_info)
{
        int err = security_shm_shmctl(NULL, SHM_INFO);
        if (!err) {
                memset(shm_info, 0, sizeof(*shm_info));
                down_read(&shm_ids(ns).rwsem);
                shm_info->used_ids = shm_ids(ns).in_use;
                shm_get_stat(ns, &shm_info->shm_rss, &shm_info->shm_swp);
                shm_info->shm_tot = ns->shm_tot;
                shm_info->swap_attempts = 0;
                shm_info->swap_successes = 0;
                err = ipc_get_maxidx(&shm_ids(ns));
                up_read(&shm_ids(ns).rwsem);
                if (err < 0)
                        err = 0;
        }
        return err;
}

static int shmctl_stat(struct ipc_namespace *ns, int shmid,
                        int cmd, struct shmid64_ds *tbuf)
{
        struct shmid_kernel *shp;
        int err;

        memset(tbuf, 0, sizeof(*tbuf));

        rcu_read_lock();
        if (cmd == SHM_STAT || cmd == SHM_STAT_ANY) {
                shp = shm_obtain_object(ns, shmid);
                if (IS_ERR(shp)) {
                        err = PTR_ERR(shp);
                        goto out_unlock;
                }
        } else { /* IPC_STAT */
                shp = shm_obtain_object_check(ns, shmid);
                if (IS_ERR(shp)) {
                        err = PTR_ERR(shp);
                        goto out_unlock;
                }
        }

        /*
         * Semantically SHM_STAT_ANY ought to be identical to
         * that functionality provided by the /proc/sysvipc/
         * interface. As such, only audit these calls and
         * do not do traditional S_IRUGO permission checks on
         * the ipc object.
         */
        if (cmd == SHM_STAT_ANY)
                audit_ipc_obj(&shp->shm_perm);
        else {
                err = -EACCES;
                if (ipcperms(ns, &shp->shm_perm, S_IRUGO))
                        goto out_unlock;
        }

        err = security_shm_shmctl(&shp->shm_perm, cmd);
        if (err)
                goto out_unlock;

        ipc_lock_object(&shp->shm_perm);

        if (!ipc_valid_object(&shp->shm_perm)) {
                ipc_unlock_object(&shp->shm_perm);
                err = -EIDRM;
                goto out_unlock;
        }

        kernel_to_ipc64_perm(&shp->shm_perm, &tbuf->shm_perm);
        tbuf->shm_segsz        = shp->shm_segsz;
        tbuf->shm_atime        = shp->shm_atim;
        tbuf->shm_dtime        = shp->shm_dtim;
        tbuf->shm_ctime        = shp->shm_ctim;
#ifndef CONFIG_64BIT
        tbuf->shm_atime_high = shp->shm_atim >> 32;
        tbuf->shm_dtime_high = shp->shm_dtim >> 32;
        tbuf->shm_ctime_high = shp->shm_ctim >> 32;
#endif
        tbuf->shm_cpid        = pid_vnr(shp->shm_cprid);
        tbuf->shm_lpid        = pid_vnr(shp->shm_lprid);
        tbuf->shm_nattch = shp->shm_nattch;

        if (cmd == IPC_STAT) {
                /*
                 * As defined in SUS:
                 * Return 0 on success
                 */
                err = 0;
        } else {
                /*
                 * SHM_STAT and SHM_STAT_ANY (both Linux specific)
                 * Return the full id, including the sequence number
                 */
                err = shp->shm_perm.id;
        }

        ipc_unlock_object(&shp->shm_perm);
out_unlock:
        rcu_read_unlock();
        return err;
}

static int shmctl_do_lock(struct ipc_namespace *ns, int shmid, int cmd)
{
        struct shmid_kernel *shp;
        struct file *shm_file;
        int err;

        rcu_read_lock();
        shp = shm_obtain_object_check(ns, shmid);
        if (IS_ERR(shp)) {
                err = PTR_ERR(shp);
                goto out_unlock1;
        }

        audit_ipc_obj(&(shp->shm_perm));
        err = security_shm_shmctl(&shp->shm_perm, cmd);
        if (err)
                goto out_unlock1;

        ipc_lock_object(&shp->shm_perm);

        /* check if shm_destroy() is tearing down shp */
        if (!ipc_valid_object(&shp->shm_perm)) {
                err = -EIDRM;
                goto out_unlock0;
        }

        if (!ns_capable(ns->user_ns, CAP_IPC_LOCK)) {
                kuid_t euid = current_euid();

                if (!uid_eq(euid, shp->shm_perm.uid) &&
                    !uid_eq(euid, shp->shm_perm.cuid)) {
                        err = -EPERM;
                        goto out_unlock0;
                }
                if (cmd == SHM_LOCK && !rlimit(RLIMIT_MEMLOCK)) {
                        err = -EPERM;
                        goto out_unlock0;
                }
        }

        shm_file = shp->shm_file;
        if (is_file_hugepages(shm_file))
                goto out_unlock0;

        if (cmd == SHM_LOCK) {
                struct ucounts *ucounts = current_ucounts();

                err = shmem_lock(shm_file, 1, ucounts);
                if (!err && !(shp->shm_perm.mode & SHM_LOCKED)) {
                        shp->shm_perm.mode |= SHM_LOCKED;
                        shp->mlock_ucounts = ucounts;
                }
                goto out_unlock0;
        }

        /* SHM_UNLOCK */
        if (!(shp->shm_perm.mode & SHM_LOCKED))
                goto out_unlock0;
        shmem_lock(shm_file, 0, shp->mlock_ucounts);
        shp->shm_perm.mode &= ~SHM_LOCKED;
        shp->mlock_ucounts = NULL;
        get_file(shm_file);
        ipc_unlock_object(&shp->shm_perm);
        rcu_read_unlock();
        shmem_unlock_mapping(shm_file->f_mapping);

        fput(shm_file);
        return err;

out_unlock0:
        ipc_unlock_object(&shp->shm_perm);
out_unlock1:
        rcu_read_unlock();
        return err;
}

static long ksys_shmctl(int shmid, int cmd, struct shmid_ds __user *buf, int version)
{
        int err;
        struct ipc_namespace *ns;
        struct shmid64_ds sem64;

        if (cmd < 0 || shmid < 0)
                return -EINVAL;

        ns = current->nsproxy->ipc_ns;

        switch (cmd) {
        case IPC_INFO: {
                struct shminfo64 shminfo;
                err = shmctl_ipc_info(ns, &shminfo);
                if (err < 0)
                        return err;
                if (copy_shminfo_to_user(buf, &shminfo, version))
                        err = -EFAULT;
                return err;
        }
        case SHM_INFO: {
                struct shm_info shm_info;
                err = shmctl_shm_info(ns, &shm_info);
                if (err < 0)
                        return err;
                if (copy_to_user(buf, &shm_info, sizeof(shm_info)))
                        err = -EFAULT;
                return err;
        }
        case SHM_STAT:
        case SHM_STAT_ANY:
        case IPC_STAT: {
                err = shmctl_stat(ns, shmid, cmd, &sem64);
                if (err < 0)
                        return err;
                if (copy_shmid_to_user(buf, &sem64, version))
                        err = -EFAULT;
                return err;
        }
        case IPC_SET:
                if (copy_shmid_from_user(&sem64, buf, version))
                        return -EFAULT;
                fallthrough;
        case IPC_RMID:
                return shmctl_down(ns, shmid, cmd, &sem64);
        case SHM_LOCK:
        case SHM_UNLOCK:
                return shmctl_do_lock(ns, shmid, cmd);
        default:
                return -EINVAL;
        }
}

SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
{
        return ksys_shmctl(shmid, cmd, buf, IPC_64);
}

#ifdef CONFIG_ARCH_WANT_IPC_PARSE_VERSION
long ksys_old_shmctl(int shmid, int cmd, struct shmid_ds __user *buf)
{
        int version = ipc_parse_version(&cmd);

        return ksys_shmctl(shmid, cmd, buf, version);
}

SYSCALL_DEFINE3(old_shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
{
        return ksys_old_shmctl(shmid, cmd, buf);
}
#endif

#ifdef CONFIG_COMPAT

struct compat_shmid_ds {
        struct compat_ipc_perm shm_perm;
        int shm_segsz;
        old_time32_t shm_atime;
        old_time32_t shm_dtime;
        old_time32_t shm_ctime;
        compat_ipc_pid_t shm_cpid;
        compat_ipc_pid_t shm_lpid;
        unsigned short shm_nattch;
        unsigned short shm_unused;
        compat_uptr_t shm_unused2;
        compat_uptr_t shm_unused3;
};

struct compat_shminfo64 {
        compat_ulong_t shmmax;
        compat_ulong_t shmmin;
        compat_ulong_t shmmni;
        compat_ulong_t shmseg;
        compat_ulong_t shmall;
        compat_ulong_t __unused1;
        compat_ulong_t __unused2;
        compat_ulong_t __unused3;
        compat_ulong_t __unused4;
};

struct compat_shm_info {
        compat_int_t used_ids;
        compat_ulong_t shm_tot, shm_rss, shm_swp;
        compat_ulong_t swap_attempts, swap_successes;
};

static int copy_compat_shminfo_to_user(void __user *buf, struct shminfo64 *in,
                                        int version)
{
        if (in->shmmax > INT_MAX)
                in->shmmax = INT_MAX;
        if (version == IPC_64) {
                struct compat_shminfo64 info;
                memset(&info, 0, sizeof(info));
                info.shmmax = in->shmmax;
                info.shmmin = in->shmmin;
                info.shmmni = in->shmmni;
                info.shmseg = in->shmseg;
                info.shmall = in->shmall;
                return copy_to_user(buf, &info, sizeof(info));
        } else {
                struct shminfo info;
                memset(&info, 0, sizeof(info));
                info.shmmax = in->shmmax;
                info.shmmin = in->shmmin;
                info.shmmni = in->shmmni;
                info.shmseg = in->shmseg;
                info.shmall = in->shmall;
                return copy_to_user(buf, &info, sizeof(info));
        }
}

static int put_compat_shm_info(struct shm_info *ip,
                                struct compat_shm_info __user *uip)
{
        struct compat_shm_info info;

        memset(&info, 0, sizeof(info));
        info.used_ids = ip->used_ids;
        info.shm_tot = ip->shm_tot;
        info.shm_rss = ip->shm_rss;
        info.shm_swp = ip->shm_swp;
        info.swap_attempts = ip->swap_attempts;
        info.swap_successes = ip->swap_successes;
        return copy_to_user(uip, &info, sizeof(info));
}

static int copy_compat_shmid_to_user(void __user *buf, struct shmid64_ds *in,
                                        int version)
{
        if (version == IPC_64) {
                struct compat_shmid64_ds v;
                memset(&v, 0, sizeof(v));
                to_compat_ipc64_perm(&v.shm_perm, &in->shm_perm);
                v.shm_atime         = lower_32_bits(in->shm_atime);
                v.shm_atime_high = upper_32_bits(in->shm_atime);
                v.shm_dtime         = lower_32_bits(in->shm_dtime);
                v.shm_dtime_high = upper_32_bits(in->shm_dtime);
                v.shm_ctime         = lower_32_bits(in->shm_ctime);
                v.shm_ctime_high = upper_32_bits(in->shm_ctime);
                v.shm_segsz = in->shm_segsz;
                v.shm_nattch = in->shm_nattch;
                v.shm_cpid = in->shm_cpid;
                v.shm_lpid = in->shm_lpid;
                return copy_to_user(buf, &v, sizeof(v));
        } else {
                struct compat_shmid_ds v;
                memset(&v, 0, sizeof(v));
                to_compat_ipc_perm(&v.shm_perm, &in->shm_perm);
                v.shm_perm.key = in->shm_perm.key;
                v.shm_atime = in->shm_atime;
                v.shm_dtime = in->shm_dtime;
                v.shm_ctime = in->shm_ctime;
                v.shm_segsz = in->shm_segsz;
                v.shm_nattch = in->shm_nattch;
                v.shm_cpid = in->shm_cpid;
                v.shm_lpid = in->shm_lpid;
                return copy_to_user(buf, &v, sizeof(v));
        }
}

static int copy_compat_shmid_from_user(struct shmid64_ds *out, void __user *buf,
                                        int version)
{
        memset(out, 0, sizeof(*out));
        if (version == IPC_64) {
                struct compat_shmid64_ds __user *p = buf;
                return get_compat_ipc64_perm(&out->shm_perm, &p->shm_perm);
        } else {
                struct compat_shmid_ds __user *p = buf;
                return get_compat_ipc_perm(&out->shm_perm, &p->shm_perm);
        }
}

static long compat_ksys_shmctl(int shmid, int cmd, void __user *uptr, int version)
{
        struct ipc_namespace *ns;
        struct shmid64_ds sem64;
        int err;

        ns = current->nsproxy->ipc_ns;

        if (cmd < 0 || shmid < 0)
                return -EINVAL;

        switch (cmd) {
        case IPC_INFO: {
                struct shminfo64 shminfo;
                err = shmctl_ipc_info(ns, &shminfo);
                if (err < 0)
                        return err;
                if (copy_compat_shminfo_to_user(uptr, &shminfo, version))
                        err = -EFAULT;
                return err;
        }
        case SHM_INFO: {
                struct shm_info shm_info;
                err = shmctl_shm_info(ns, &shm_info);
                if (err < 0)
                        return err;
                if (put_compat_shm_info(&shm_info, uptr))
                        err = -EFAULT;
                return err;
        }
        case IPC_STAT:
        case SHM_STAT_ANY:
        case SHM_STAT:
                err = shmctl_stat(ns, shmid, cmd, &sem64);
                if (err < 0)
                        return err;
                if (copy_compat_shmid_to_user(uptr, &sem64, version))
                        err = -EFAULT;
                return err;

        case IPC_SET:
                if (copy_compat_shmid_from_user(&sem64, uptr, version))
                        return -EFAULT;
                fallthrough;
        case IPC_RMID:
                return shmctl_down(ns, shmid, cmd, &sem64);
        case SHM_LOCK:
        case SHM_UNLOCK:
                return shmctl_do_lock(ns, shmid, cmd);
        default:
                return -EINVAL;
        }
        return err;
}

COMPAT_SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, void __user *, uptr)
{
        return compat_ksys_shmctl(shmid, cmd, uptr, IPC_64);
}

#ifdef CONFIG_ARCH_WANT_COMPAT_IPC_PARSE_VERSION
long compat_ksys_old_shmctl(int shmid, int cmd, void __user *uptr)
{
        int version = compat_ipc_parse_version(&cmd);

        return compat_ksys_shmctl(shmid, cmd, uptr, version);
}

COMPAT_SYSCALL_DEFINE3(old_shmctl, int, shmid, int, cmd, void __user *, uptr)
{
        return compat_ksys_old_shmctl(shmid, cmd, uptr);
}
#endif
#endif

/*
 * Fix shmaddr, allocate descriptor, map shm, add attach descriptor to lists.
 *
 * NOTE! Despite the name, this is NOT a direct system call entrypoint. The
 * "raddr" thing points to kernel space, and there has to be a wrapper around
 * this.
 */
long do_shmat(int shmid, char __user *shmaddr, int shmflg,
              ulong *raddr, unsigned long shmlba)
{
        struct shmid_kernel *shp;
        unsigned long addr = (unsigned long)shmaddr;
        unsigned long size;
        struct file *file, *base;
        int    err;
        unsigned long flags = MAP_SHARED;
        unsigned long prot;
        int acc_mode;
        struct ipc_namespace *ns;
        struct shm_file_data *sfd;
        int f_flags;
        unsigned long populate = 0;

        err = -EINVAL;
        if (shmid < 0)
                goto out;

        if (addr) {
                if (addr & (shmlba - 1)) {
                        if (shmflg & SHM_RND) {
                                addr &= ~(shmlba - 1);  /* round down */

                                /*
                                 * Ensure that the round-down is non-nil
                                 * when remapping. This can happen for
                                 * cases when addr < shmlba.
                                 */
                                if (!addr && (shmflg & SHM_REMAP))
                                        goto out;
                        } else
#ifndef __ARCH_FORCE_SHMLBA
                                if (addr & ~PAGE_MASK)
#endif
                                        goto out;
                }

                flags |= MAP_FIXED;
        } else if ((shmflg & SHM_REMAP))
                goto out;

        if (shmflg & SHM_RDONLY) {
                prot = PROT_READ;
                acc_mode = S_IRUGO;
                f_flags = O_RDONLY;
        } else {
                prot = PROT_READ | PROT_WRITE;
                acc_mode = S_IRUGO | S_IWUGO;
                f_flags = O_RDWR;
        }
        if (shmflg & SHM_EXEC) {
                prot |= PROT_EXEC;
                acc_mode |= S_IXUGO;
        }

        /*
         * We cannot rely on the fs check since SYSV IPC does have an
         * additional creator id...
         */
        ns = current->nsproxy->ipc_ns;
        rcu_read_lock();
        shp = shm_obtain_object_check(ns, shmid);
        if (IS_ERR(shp)) {
                err = PTR_ERR(shp);
                goto out_unlock;
        }

        err = -EACCES;
        if (ipcperms(ns, &shp->shm_perm, acc_mode))
                goto out_unlock;

        err = security_shm_shmat(&shp->shm_perm, shmaddr, shmflg);
        if (err)
                goto out_unlock;

        ipc_lock_object(&shp->shm_perm);

        /* check if shm_destroy() is tearing down shp */
        if (!ipc_valid_object(&shp->shm_perm)) {
                ipc_unlock_object(&shp->shm_perm);
                err = -EIDRM;
                goto out_unlock;
        }

        /*
         * We need to take a reference to the real shm file to prevent the
         * pointer from becoming stale in cases where the lifetime of the outer
         * file extends beyond that of the shm segment.  It's not usually
         * possible, but it can happen during remap_file_pages() emulation as
         * that unmaps the memory, then does ->mmap() via file reference only.
         * We'll deny the ->mmap() if the shm segment was since removed, but to
         * detect shm ID reuse we need to compare the file pointers.
         */
        base = get_file(shp->shm_file);
        shp->shm_nattch++;
        size = i_size_read(file_inode(base));
        ipc_unlock_object(&shp->shm_perm);
        rcu_read_unlock();

        err = -ENOMEM;
        sfd = kzalloc(sizeof(*sfd), GFP_KERNEL);
        if (!sfd) {
                fput(base);
                goto out_nattch;
        }

        file = alloc_file_clone(base, f_flags,
                          is_file_hugepages(base) ?
                                &shm_file_operations_huge :
                                &shm_file_operations);
        err = PTR_ERR(file);
        if (IS_ERR(file)) {
                kfree(sfd);
                fput(base);
                goto out_nattch;
        }

        sfd->id = shp->shm_perm.id;
        sfd->ns = get_ipc_ns(ns);
        sfd->file = base;
        sfd->vm_ops = NULL;
        file->private_data = sfd;

        err = security_mmap_file(file, prot, flags);
        if (err)
                goto out_fput;

        if (mmap_write_lock_killable(current->mm)) {
                err = -EINTR;
                goto out_fput;
        }

        if (addr && !(shmflg & SHM_REMAP)) {
                err = -EINVAL;
                if (addr + size < addr)
                        goto invalid;

                if (find_vma_intersection(current->mm, addr, addr + size))
                        goto invalid;
        }

        addr = do_mmap(file, addr, size, prot, flags, 0, 0, &populate, NULL);
        *raddr = addr;
        err = 0;
        if (IS_ERR_VALUE(addr))
                err = (long)addr;
invalid:
        mmap_write_unlock(current->mm);
        if (populate)
                mm_populate(addr, populate);

out_fput:
        fput(file);

out_nattch:
        down_write(&shm_ids(ns).rwsem);
        shp = shm_lock(ns, shmid);
        shp->shm_nattch--;

        if (shm_may_destroy(shp))
                shm_destroy(ns, shp);
        else
                shm_unlock(shp);
        up_write(&shm_ids(ns).rwsem);
        return err;

out_unlock:
        rcu_read_unlock();
out:
        return err;
}

SYSCALL_DEFINE3(shmat, int, shmid, char __user *, shmaddr, int, shmflg)
{
        unsigned long ret;
        long err;

        err = do_shmat(shmid, shmaddr, shmflg, &ret, SHMLBA);
        if (err)
                return err;
        force_successful_syscall_return();
        return (long)ret;
}

#ifdef CONFIG_COMPAT

#ifndef COMPAT_SHMLBA
#define COMPAT_SHMLBA        SHMLBA
#endif

COMPAT_SYSCALL_DEFINE3(shmat, int, shmid, compat_uptr_t, shmaddr, int, shmflg)
{
        unsigned long ret;
        long err;

        err = do_shmat(shmid, compat_ptr(shmaddr), shmflg, &ret, COMPAT_SHMLBA);
        if (err)
                return err;
        force_successful_syscall_return();
        return (long)ret;
}
#endif

/*
 * detach and kill segment if marked destroyed.
 * The work is done in shm_close.
 */
long ksys_shmdt(char __user *shmaddr)
{
        struct mm_struct *mm = current->mm;
        struct vm_area_struct *vma;
        unsigned long addr = (unsigned long)shmaddr;
        int retval = -EINVAL;
#ifdef CONFIG_MMU
        loff_t size = 0;
        struct file *file;
        VMA_ITERATOR(vmi, mm, addr);
#endif

        if (addr & ~PAGE_MASK)
                return retval;

        if (mmap_write_lock_killable(mm))
                return -EINTR;

        /*
         * This function tries to be smart and unmap shm segments that
         * were modified by partial mlock or munmap calls:
         * - It first determines the size of the shm segment that should be
         *   unmapped: It searches for a vma that is backed by shm and that
         *   started at address shmaddr. It records it's size and then unmaps
         *   it.
         * - Then it unmaps all shm vmas that started at shmaddr and that
         *   are within the initially determined size and that are from the
         *   same shm segment from which we determined the size.
         * Errors from do_munmap are ignored: the function only fails if
         * it's called with invalid parameters or if it's called to unmap
         * a part of a vma. Both calls in this function are for full vmas,
         * the parameters are directly copied from the vma itself and always
         * valid - therefore do_munmap cannot fail. (famous last words?)
         */
        /*
         * If it had been mremap()'d, the starting address would not
         * match the usual checks anyway. So assume all vma's are
         * above the starting address given.
         */

#ifdef CONFIG_MMU
        for_each_vma(vmi, vma) {
                /*
                 * Check if the starting address would match, i.e. it's
                 * a fragment created by mprotect() and/or munmap(), or it
                 * otherwise it starts at this address with no hassles.
                 */
                if ((vma->vm_ops == &shm_vm_ops) &&
                        (vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) {

                        /*
                         * Record the file of the shm segment being
                         * unmapped.  With mremap(), someone could place
                         * page from another segment but with equal offsets
                         * in the range we are unmapping.
                         */
                        file = vma->vm_file;
                        size = i_size_read(file_inode(vma->vm_file));
                        do_vmi_align_munmap(&vmi, vma, mm, vma->vm_start,
                                            vma->vm_end, NULL, false);
                        /*
                         * We discovered the size of the shm segment, so
                         * break out of here and fall through to the next
                         * loop that uses the size information to stop
                         * searching for matching vma's.
                         */
                        retval = 0;
                        vma = vma_next(&vmi);
                        break;
                }
        }

        /*
         * We need look no further than the maximum address a fragment
         * could possibly have landed at. Also cast things to loff_t to
         * prevent overflows and make comparisons vs. equal-width types.
         */
        size = PAGE_ALIGN(size);
        while (vma && (loff_t)(vma->vm_end - addr) <= size) {
                /* finding a matching vma now does not alter retval */
                if ((vma->vm_ops == &shm_vm_ops) &&
                    ((vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) &&
                    (vma->vm_file == file)) {
                        do_vmi_align_munmap(&vmi, vma, mm, vma->vm_start,
                                            vma->vm_end, NULL, false);
                }

                vma = vma_next(&vmi);
        }

#else        /* CONFIG_MMU */
        vma = vma_lookup(mm, addr);
        /* under NOMMU conditions, the exact address to be destroyed must be
         * given
         */
        if (vma && vma->vm_start == addr && vma->vm_ops == &shm_vm_ops) {
                do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL);
                retval = 0;
        }

#endif

        mmap_write_unlock(mm);
        return retval;
}

SYSCALL_DEFINE1(shmdt, char __user *, shmaddr)
{
        return ksys_shmdt(shmaddr);
}

#ifdef CONFIG_PROC_FS
static int sysvipc_shm_proc_show(struct seq_file *s, void *it)
{
        struct pid_namespace *pid_ns = ipc_seq_pid_ns(s);
        struct user_namespace *user_ns = seq_user_ns(s);
        struct kern_ipc_perm *ipcp = it;
        struct shmid_kernel *shp;
        unsigned long rss = 0, swp = 0;

        shp = container_of(ipcp, struct shmid_kernel, shm_perm);
        shm_add_rss_swap(shp, &rss, &swp);

#if BITS_PER_LONG <= 32
#define SIZE_SPEC "%10lu"
#else
#define SIZE_SPEC "%21lu"
#endif

        seq_printf(s,
                   "%10d %10d  %4o " SIZE_SPEC " %5u %5u  "
                   "%5lu %5u %5u %5u %5u %10llu %10llu %10llu "
                   SIZE_SPEC " " SIZE_SPEC "\n",
                   shp->shm_perm.key,
                   shp->shm_perm.id,
                   shp->shm_perm.mode,
                   shp->shm_segsz,
                   pid_nr_ns(shp->shm_cprid, pid_ns),
                   pid_nr_ns(shp->shm_lprid, pid_ns),
                   shp->shm_nattch,
                   from_kuid_munged(user_ns, shp->shm_perm.uid),
                   from_kgid_munged(user_ns, shp->shm_perm.gid),
                   from_kuid_munged(user_ns, shp->shm_perm.cuid),
                   from_kgid_munged(user_ns, shp->shm_perm.cgid),
                   shp->shm_atim,
                   shp->shm_dtim,
                   shp->shm_ctim,
                   rss * PAGE_SIZE,
                   swp * PAGE_SIZE);

        return 0;
}
#endif


























































































































































































































































































































































































   28 














   28 
























































   28 













   28 






























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
// SPDX-License-Identifier: GPL-2.0-or-later
/*        6LoWPAN fragment reassembly
 *
 *        Authors:
 *        Alexander Aring                <aar@pengutronix.de>
 *
 *        Based on: net/ipv6/reassembly.c
 */

#define pr_fmt(fmt) "6LoWPAN: " fmt

#include <linux/net.h>
#include <linux/list.h>
#include <linux/netdevice.h>
#include <linux/random.h>
#include <linux/jhash.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
#include <linux/export.h>

#include <net/ieee802154_netdev.h>
#include <net/6lowpan.h>
#include <net/ipv6_frag.h>
#include <net/inet_frag.h>
#include <net/ip.h>

#include "6lowpan_i.h"

static const char lowpan_frags_cache_name[] = "lowpan-frags";

static struct inet_frags lowpan_frags;

static int lowpan_frag_reasm(struct lowpan_frag_queue *fq, struct sk_buff *skb,
                             struct sk_buff *prev, struct net_device *ldev,
                             int *refs);

static void lowpan_frag_init(struct inet_frag_queue *q, const void *a)
{
        const struct frag_lowpan_compare_key *key = a;

        BUILD_BUG_ON(sizeof(*key) > sizeof(q->key));
        memcpy(&q->key, key, sizeof(*key));
}

static void lowpan_frag_expire(struct timer_list *t)
{
        struct inet_frag_queue *frag = from_timer(frag, t, timer);
        struct frag_queue *fq;
        int refs = 1;

        fq = container_of(frag, struct frag_queue, q);

        spin_lock(&fq->q.lock);

        if (fq->q.flags & INET_FRAG_COMPLETE)
                goto out;

        inet_frag_kill(&fq->q, &refs);
out:
        spin_unlock(&fq->q.lock);
        inet_frag_putn(&fq->q, refs);
}

static inline struct lowpan_frag_queue *
fq_find(struct net *net, const struct lowpan_802154_cb *cb,
        const struct ieee802154_addr *src,
        const struct ieee802154_addr *dst)
{
        struct netns_ieee802154_lowpan *ieee802154_lowpan =
                net_ieee802154_lowpan(net);
        struct frag_lowpan_compare_key key = {};
        struct inet_frag_queue *q;

        key.tag = cb->d_tag;
        key.d_size = cb->d_size;
        key.src = *src;
        key.dst = *dst;

        q = inet_frag_find(ieee802154_lowpan->fqdir, &key);
        if (!q)
                return NULL;

        return container_of(q, struct lowpan_frag_queue, q);
}

static int lowpan_frag_queue(struct lowpan_frag_queue *fq,
                             struct sk_buff *skb, u8 frag_type,
                             int *refs)
{
        struct sk_buff *prev_tail;
        struct net_device *ldev;
        int end, offset, err;

        /* inet_frag_queue_* functions use skb->cb; see struct ipfrag_skb_cb
         * in inet_fragment.c
         */
        BUILD_BUG_ON(sizeof(struct lowpan_802154_cb) > sizeof(struct inet_skb_parm));
        BUILD_BUG_ON(sizeof(struct lowpan_802154_cb) > sizeof(struct inet6_skb_parm));

        if (fq->q.flags & INET_FRAG_COMPLETE)
                goto err;

        offset = lowpan_802154_cb(skb)->d_offset << 3;
        end = lowpan_802154_cb(skb)->d_size;

        /* Is this the final fragment? */
        if (offset + skb->len == end) {
                /* If we already have some bits beyond end
                 * or have different end, the segment is corrupted.
                 */
                if (end < fq->q.len ||
                    ((fq->q.flags & INET_FRAG_LAST_IN) && end != fq->q.len))
                        goto err;
                fq->q.flags |= INET_FRAG_LAST_IN;
                fq->q.len = end;
        } else {
                if (end > fq->q.len) {
                        /* Some bits beyond end -> corruption. */
                        if (fq->q.flags & INET_FRAG_LAST_IN)
                                goto err;
                        fq->q.len = end;
                }
        }

        ldev = skb->dev;
        if (ldev)
                skb->dev = NULL;
        barrier();

        prev_tail = fq->q.fragments_tail;
        err = inet_frag_queue_insert(&fq->q, skb, offset, end);
        if (err)
                goto err;

        fq->q.stamp = skb->tstamp;
        fq->q.tstamp_type = skb->tstamp_type;
        if (frag_type == LOWPAN_DISPATCH_FRAG1)
                fq->q.flags |= INET_FRAG_FIRST_IN;

        fq->q.meat += skb->len;
        add_frag_mem_limit(fq->q.fqdir, skb->truesize);

        if (fq->q.flags == (INET_FRAG_FIRST_IN | INET_FRAG_LAST_IN) &&
            fq->q.meat == fq->q.len) {
                int res;
                unsigned long orefdst = skb->_skb_refdst;

                skb->_skb_refdst = 0UL;
                res = lowpan_frag_reasm(fq, skb, prev_tail, ldev, refs);
                skb->_skb_refdst = orefdst;
                return res;
        }
        skb_dst_drop(skb);

        return -1;
err:
        kfree_skb(skb);
        return -1;
}

/*        Check if this packet is complete.
 *
 *        It is called with locked fq, and caller must check that
 *        queue is eligible for reassembly i.e. it is not COMPLETE,
 *        the last and the first frames arrived and all the bits are here.
 */
static int lowpan_frag_reasm(struct lowpan_frag_queue *fq, struct sk_buff *skb,
                             struct sk_buff *prev_tail, struct net_device *ldev,
                             int *refs)
{
        void *reasm_data;

        inet_frag_kill(&fq->q, refs);

        reasm_data = inet_frag_reasm_prepare(&fq->q, skb, prev_tail);
        if (!reasm_data)
                goto out_oom;
        inet_frag_reasm_finish(&fq->q, skb, reasm_data, false);

        skb->dev = ldev;
        skb->tstamp = fq->q.stamp;
        fq->q.rb_fragments = RB_ROOT;
        fq->q.fragments_tail = NULL;
        fq->q.last_run_head = NULL;

        return 1;
out_oom:
        net_dbg_ratelimited("lowpan_frag_reasm: no memory for reassembly\n");
        return -1;
}

static int lowpan_frag_rx_handlers_result(struct sk_buff *skb,
                                          lowpan_rx_result res)
{
        switch (res) {
        case RX_QUEUED:
                return NET_RX_SUCCESS;
        case RX_CONTINUE:
                /* nobody cared about this packet */
                net_warn_ratelimited("%s: received unknown dispatch\n",
                                     __func__);

                fallthrough;
        default:
                /* all others failure */
                return NET_RX_DROP;
        }
}

static lowpan_rx_result lowpan_frag_rx_h_iphc(struct sk_buff *skb)
{
        int ret;

        if (!lowpan_is_iphc(*skb_network_header(skb)))
                return RX_CONTINUE;

        ret = lowpan_iphc_decompress(skb);
        if (ret < 0)
                return RX_DROP;

        return RX_QUEUED;
}

static int lowpan_invoke_frag_rx_handlers(struct sk_buff *skb)
{
        lowpan_rx_result res;

#define CALL_RXH(rxh)                        \
        do {                                \
                res = rxh(skb);        \
                if (res != RX_CONTINUE)        \
                        goto rxh_next;        \
        } while (0)

        /* likely at first */
        CALL_RXH(lowpan_frag_rx_h_iphc);
        CALL_RXH(lowpan_rx_h_ipv6);

rxh_next:
        return lowpan_frag_rx_handlers_result(skb, res);
#undef CALL_RXH
}

#define LOWPAN_FRAG_DGRAM_SIZE_HIGH_MASK        0x07
#define LOWPAN_FRAG_DGRAM_SIZE_HIGH_SHIFT        8

static int lowpan_get_cb(struct sk_buff *skb, u8 frag_type,
                         struct lowpan_802154_cb *cb)
{
        bool fail;
        u8 high = 0, low = 0;
        __be16 d_tag = 0;

        fail = lowpan_fetch_skb(skb, &high, 1);
        fail |= lowpan_fetch_skb(skb, &low, 1);
        /* remove the dispatch value and use first three bits as high value
         * for the datagram size
         */
        cb->d_size = (high & LOWPAN_FRAG_DGRAM_SIZE_HIGH_MASK) <<
                LOWPAN_FRAG_DGRAM_SIZE_HIGH_SHIFT | low;
        fail |= lowpan_fetch_skb(skb, &d_tag, 2);
        cb->d_tag = ntohs(d_tag);

        if (frag_type == LOWPAN_DISPATCH_FRAGN) {
                fail |= lowpan_fetch_skb(skb, &cb->d_offset, 1);
        } else {
                skb_reset_network_header(skb);
                cb->d_offset = 0;
                /* check if datagram_size has ipv6hdr on FRAG1 */
                fail |= cb->d_size < sizeof(struct ipv6hdr);
                /* check if we can dereference the dispatch value */
                fail |= !skb->len;
        }

        if (unlikely(fail))
                return -EIO;

        return 0;
}

int lowpan_frag_rcv(struct sk_buff *skb, u8 frag_type)
{
        struct lowpan_frag_queue *fq;
        struct net *net = dev_net(skb->dev);
        struct lowpan_802154_cb *cb = lowpan_802154_cb(skb);
        struct ieee802154_hdr hdr = {};
        int err;

        if (ieee802154_hdr_peek_addrs(skb, &hdr) < 0)
                goto err;

        err = lowpan_get_cb(skb, frag_type, cb);
        if (err < 0)
                goto err;

        if (frag_type == LOWPAN_DISPATCH_FRAG1) {
                err = lowpan_invoke_frag_rx_handlers(skb);
                if (err == NET_RX_DROP)
                        goto err;
        }

        if (cb->d_size > IPV6_MIN_MTU) {
                net_warn_ratelimited("lowpan_frag_rcv: datagram size exceeds MTU\n");
                goto err;
        }

        rcu_read_lock();
        fq = fq_find(net, cb, &hdr.source, &hdr.dest);
        if (fq != NULL) {
                int ret, refs = 0;

                spin_lock(&fq->q.lock);
                ret = lowpan_frag_queue(fq, skb, frag_type, &refs);
                spin_unlock(&fq->q.lock);

                rcu_read_unlock();
                inet_frag_putn(&fq->q, refs);
                return ret;
        }
        rcu_read_unlock();

err:
        kfree_skb(skb);
        return -1;
}

#ifdef CONFIG_SYSCTL

static struct ctl_table lowpan_frags_ns_ctl_table[] = {
        {
                .procname        = "6lowpanfrag_high_thresh",
                .maxlen                = sizeof(unsigned long),
                .mode                = 0644,
                .proc_handler        = proc_doulongvec_minmax,
        },
        {
                .procname        = "6lowpanfrag_low_thresh",
                .maxlen                = sizeof(unsigned long),
                .mode                = 0644,
                .proc_handler        = proc_doulongvec_minmax,
        },
        {
                .procname        = "6lowpanfrag_time",
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
};

/* secret interval has been deprecated */
static int lowpan_frags_secret_interval_unused;
static struct ctl_table lowpan_frags_ctl_table[] = {
        {
                .procname        = "6lowpanfrag_secret_interval",
                .data                = &lowpan_frags_secret_interval_unused,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
};

static int __net_init lowpan_frags_ns_sysctl_register(struct net *net)
{
        struct ctl_table *table;
        struct ctl_table_header *hdr;
        struct netns_ieee802154_lowpan *ieee802154_lowpan =
                net_ieee802154_lowpan(net);
        size_t table_size = ARRAY_SIZE(lowpan_frags_ns_ctl_table);

        table = lowpan_frags_ns_ctl_table;
        if (!net_eq(net, &init_net)) {
                table = kmemdup(table, sizeof(lowpan_frags_ns_ctl_table),
                                GFP_KERNEL);
                if (table == NULL)
                        goto err_alloc;

                /* Don't export sysctls to unprivileged users */
                if (net->user_ns != &init_user_ns)
                        table_size = 0;
        }

        table[0].data        = &ieee802154_lowpan->fqdir->high_thresh;
        table[0].extra1        = &ieee802154_lowpan->fqdir->low_thresh;
        table[1].data        = &ieee802154_lowpan->fqdir->low_thresh;
        table[1].extra2        = &ieee802154_lowpan->fqdir->high_thresh;
        table[2].data        = &ieee802154_lowpan->fqdir->timeout;

        hdr = register_net_sysctl_sz(net, "net/ieee802154/6lowpan", table,
                                     table_size);
        if (hdr == NULL)
                goto err_reg;

        ieee802154_lowpan->sysctl.frags_hdr = hdr;
        return 0;

err_reg:
        if (!net_eq(net, &init_net))
                kfree(table);
err_alloc:
        return -ENOMEM;
}

static void __net_exit lowpan_frags_ns_sysctl_unregister(struct net *net)
{
        const struct ctl_table *table;
        struct netns_ieee802154_lowpan *ieee802154_lowpan =
                net_ieee802154_lowpan(net);

        table = ieee802154_lowpan->sysctl.frags_hdr->ctl_table_arg;
        unregister_net_sysctl_table(ieee802154_lowpan->sysctl.frags_hdr);
        if (!net_eq(net, &init_net))
                kfree(table);
}

static struct ctl_table_header *lowpan_ctl_header;

static int __init lowpan_frags_sysctl_register(void)
{
        lowpan_ctl_header = register_net_sysctl(&init_net,
                                                "net/ieee802154/6lowpan",
                                                lowpan_frags_ctl_table);
        return lowpan_ctl_header == NULL ? -ENOMEM : 0;
}

static void lowpan_frags_sysctl_unregister(void)
{
        unregister_net_sysctl_table(lowpan_ctl_header);
}
#else
static inline int lowpan_frags_ns_sysctl_register(struct net *net)
{
        return 0;
}

static inline void lowpan_frags_ns_sysctl_unregister(struct net *net)
{
}

static inline int __init lowpan_frags_sysctl_register(void)
{
        return 0;
}

static inline void lowpan_frags_sysctl_unregister(void)
{
}
#endif

static int __net_init lowpan_frags_init_net(struct net *net)
{
        struct netns_ieee802154_lowpan *ieee802154_lowpan =
                net_ieee802154_lowpan(net);
        int res;


        res = fqdir_init(&ieee802154_lowpan->fqdir, &lowpan_frags, net);
        if (res < 0)
                return res;

        ieee802154_lowpan->fqdir->high_thresh = IPV6_FRAG_HIGH_THRESH;
        ieee802154_lowpan->fqdir->low_thresh = IPV6_FRAG_LOW_THRESH;
        ieee802154_lowpan->fqdir->timeout = IPV6_FRAG_TIMEOUT;

        res = lowpan_frags_ns_sysctl_register(net);
        if (res < 0)
                fqdir_exit(ieee802154_lowpan->fqdir);
        return res;
}

static void __net_exit lowpan_frags_pre_exit_net(struct net *net)
{
        struct netns_ieee802154_lowpan *ieee802154_lowpan =
                net_ieee802154_lowpan(net);

        fqdir_pre_exit(ieee802154_lowpan->fqdir);
}

static void __net_exit lowpan_frags_exit_net(struct net *net)
{
        struct netns_ieee802154_lowpan *ieee802154_lowpan =
                net_ieee802154_lowpan(net);

        lowpan_frags_ns_sysctl_unregister(net);
        fqdir_exit(ieee802154_lowpan->fqdir);
}

static struct pernet_operations lowpan_frags_ops = {
        .init                = lowpan_frags_init_net,
        .pre_exit        = lowpan_frags_pre_exit_net,
        .exit                = lowpan_frags_exit_net,
};

static u32 lowpan_key_hashfn(const void *data, u32 len, u32 seed)
{
        return jhash2(data,
                      sizeof(struct frag_lowpan_compare_key) / sizeof(u32), seed);
}

static u32 lowpan_obj_hashfn(const void *data, u32 len, u32 seed)
{
        const struct inet_frag_queue *fq = data;

        return jhash2((const u32 *)&fq->key,
                      sizeof(struct frag_lowpan_compare_key) / sizeof(u32), seed);
}

static int lowpan_obj_cmpfn(struct rhashtable_compare_arg *arg, const void *ptr)
{
        const struct frag_lowpan_compare_key *key = arg->key;
        const struct inet_frag_queue *fq = ptr;

        return !!memcmp(&fq->key, key, sizeof(*key));
}

static const struct rhashtable_params lowpan_rhash_params = {
        .head_offset                = offsetof(struct inet_frag_queue, node),
        .hashfn                        = lowpan_key_hashfn,
        .obj_hashfn                = lowpan_obj_hashfn,
        .obj_cmpfn                = lowpan_obj_cmpfn,
        .automatic_shrinking        = true,
};

int __init lowpan_net_frag_init(void)
{
        int ret;

        lowpan_frags.constructor = lowpan_frag_init;
        lowpan_frags.destructor = NULL;
        lowpan_frags.qsize = sizeof(struct frag_queue);
        lowpan_frags.frag_expire = lowpan_frag_expire;
        lowpan_frags.frags_cache_name = lowpan_frags_cache_name;
        lowpan_frags.rhash_params = lowpan_rhash_params;
        ret = inet_frags_init(&lowpan_frags);
        if (ret)
                goto out;

        ret = lowpan_frags_sysctl_register();
        if (ret)
                goto err_sysctl;

        ret = register_pernet_subsys(&lowpan_frags_ops);
        if (ret)
                goto err_pernet;
out:
        return ret;
err_pernet:
        lowpan_frags_sysctl_unregister();
err_sysctl:
        inet_frags_fini(&lowpan_frags);
        return ret;
}

void lowpan_net_frag_exit(void)
{
        lowpan_frags_sysctl_unregister();
        unregister_pernet_subsys(&lowpan_frags_ops);
        inet_frags_fini(&lowpan_frags);
}

































































































































































































































































































































































































    7 





    7 
    7 










    7 
























































































































































    7 


    7 

























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
// SPDX-License-Identifier: GPL-2.0
/*
 *  linux/mm/mempool.c
 *
 *  memory buffer pool support. Such pools are mostly used
 *  for guaranteed, deadlock-free memory allocations during
 *  extreme VM load.
 *
 *  started by Ingo Molnar, Copyright (C) 2001
 *  debugging by David Rientjes, Copyright (C) 2015
 */

#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/highmem.h>
#include <linux/kasan.h>
#include <linux/kmemleak.h>
#include <linux/export.h>
#include <linux/mempool.h>
#include <linux/writeback.h>
#include "slab.h"

#ifdef CONFIG_SLUB_DEBUG_ON
static void poison_error(mempool_t *pool, void *element, size_t size,
                         size_t byte)
{
        const int nr = pool->curr_nr;
        const int start = max_t(int, byte - (BITS_PER_LONG / 8), 0);
        const int end = min_t(int, byte + (BITS_PER_LONG / 8), size);
        int i;

        pr_err("BUG: mempool element poison mismatch\n");
        pr_err("Mempool %p size %zu\n", pool, size);
        pr_err(" nr=%d @ %p: %s0x", nr, element, start > 0 ? "... " : "");
        for (i = start; i < end; i++)
                pr_cont("%x ", *(u8 *)(element + i));
        pr_cont("%s\n", end < size ? "..." : "");
        dump_stack();
}

static void __check_element(mempool_t *pool, void *element, size_t size)
{
        u8 *obj = element;
        size_t i;

        for (i = 0; i < size; i++) {
                u8 exp = (i < size - 1) ? POISON_FREE : POISON_END;

                if (obj[i] != exp) {
                        poison_error(pool, element, size, i);
                        return;
                }
        }
        memset(obj, POISON_INUSE, size);
}

static void check_element(mempool_t *pool, void *element)
{
        /* Skip checking: KASAN might save its metadata in the element. */
        if (kasan_enabled())
                return;

        /* Mempools backed by slab allocator */
        if (pool->free == mempool_kfree) {
                __check_element(pool, element, (size_t)pool->pool_data);
        } else if (pool->free == mempool_free_slab) {
                __check_element(pool, element, kmem_cache_size(pool->pool_data));
        } else if (pool->free == mempool_free_pages) {
                /* Mempools backed by page allocator */
                int order = (int)(long)pool->pool_data;
                void *addr = kmap_local_page((struct page *)element);

                __check_element(pool, addr, 1UL << (PAGE_SHIFT + order));
                kunmap_local(addr);
        }
}

static void __poison_element(void *element, size_t size)
{
        u8 *obj = element;

        memset(obj, POISON_FREE, size - 1);
        obj[size - 1] = POISON_END;
}

static void poison_element(mempool_t *pool, void *element)
{
        /* Skip poisoning: KASAN might save its metadata in the element. */
        if (kasan_enabled())
                return;

        /* Mempools backed by slab allocator */
        if (pool->alloc == mempool_kmalloc) {
                __poison_element(element, (size_t)pool->pool_data);
        } else if (pool->alloc == mempool_alloc_slab) {
                __poison_element(element, kmem_cache_size(pool->pool_data));
        } else if (pool->alloc == mempool_alloc_pages) {
                /* Mempools backed by page allocator */
                int order = (int)(long)pool->pool_data;
                void *addr = kmap_local_page((struct page *)element);

                __poison_element(addr, 1UL << (PAGE_SHIFT + order));
                kunmap_local(addr);
        }
}
#else /* CONFIG_SLUB_DEBUG_ON */
static inline void check_element(mempool_t *pool, void *element)
{
}
static inline void poison_element(mempool_t *pool, void *element)
{
}
#endif /* CONFIG_SLUB_DEBUG_ON */

static __always_inline bool kasan_poison_element(mempool_t *pool, void *element)
{
        if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)
                return kasan_mempool_poison_object(element);
        else if (pool->alloc == mempool_alloc_pages)
                return kasan_mempool_poison_pages(element,
                                                (unsigned long)pool->pool_data);
        return true;
}

static void kasan_unpoison_element(mempool_t *pool, void *element)
{
        if (pool->alloc == mempool_kmalloc)
                kasan_mempool_unpoison_object(element, (size_t)pool->pool_data);
        else if (pool->alloc == mempool_alloc_slab)
                kasan_mempool_unpoison_object(element,
                                              kmem_cache_size(pool->pool_data));
        else if (pool->alloc == mempool_alloc_pages)
                kasan_mempool_unpoison_pages(element,
                                             (unsigned long)pool->pool_data);
}

static __always_inline void add_element(mempool_t *pool, void *element)
{
        BUG_ON(pool->curr_nr >= pool->min_nr);
        poison_element(pool, element);
        if (kasan_poison_element(pool, element))
                pool->elements[pool->curr_nr++] = element;
}

static void *remove_element(mempool_t *pool)
{
        void *element = pool->elements[--pool->curr_nr];

        BUG_ON(pool->curr_nr < 0);
        kasan_unpoison_element(pool, element);
        check_element(pool, element);
        return element;
}

/**
 * mempool_exit - exit a mempool initialized with mempool_init()
 * @pool:      pointer to the memory pool which was initialized with
 *             mempool_init().
 *
 * Free all reserved elements in @pool and @pool itself.  This function
 * only sleeps if the free_fn() function sleeps.
 *
 * May be called on a zeroed but uninitialized mempool (i.e. allocated with
 * kzalloc()).
 */
void mempool_exit(mempool_t *pool)
{
        while (pool->curr_nr) {
                void *element = remove_element(pool);
                pool->free(element, pool->pool_data);
        }
        kfree(pool->elements);
        pool->elements = NULL;
}
EXPORT_SYMBOL(mempool_exit);

/**
 * mempool_destroy - deallocate a memory pool
 * @pool:      pointer to the memory pool which was allocated via
 *             mempool_create().
 *
 * Free all reserved elements in @pool and @pool itself.  This function
 * only sleeps if the free_fn() function sleeps.
 */
void mempool_destroy(mempool_t *pool)
{
        if (unlikely(!pool))
                return;

        mempool_exit(pool);
        kfree(pool);
}
EXPORT_SYMBOL(mempool_destroy);

int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
                      mempool_free_t *free_fn, void *pool_data,
                      gfp_t gfp_mask, int node_id)
{
        spin_lock_init(&pool->lock);
        pool->min_nr        = min_nr;
        pool->pool_data = pool_data;
        pool->alloc        = alloc_fn;
        pool->free        = free_fn;
        init_waitqueue_head(&pool->wait);

        pool->elements = kmalloc_array_node(min_nr, sizeof(void *),
                                            gfp_mask, node_id);
        if (!pool->elements)
                return -ENOMEM;

        /*
         * First pre-allocate the guaranteed number of buffers.
         */
        while (pool->curr_nr < pool->min_nr) {
                void *element;

                element = pool->alloc(gfp_mask, pool->pool_data);
                if (unlikely(!element)) {
                        mempool_exit(pool);
                        return -ENOMEM;
                }
                add_element(pool, element);
        }

        return 0;
}
EXPORT_SYMBOL(mempool_init_node);

/**
 * mempool_init - initialize a memory pool
 * @pool:      pointer to the memory pool that should be initialized
 * @min_nr:    the minimum number of elements guaranteed to be
 *             allocated for this pool.
 * @alloc_fn:  user-defined element-allocation function.
 * @free_fn:   user-defined element-freeing function.
 * @pool_data: optional private data available to the user-defined functions.
 *
 * Like mempool_create(), but initializes the pool in (i.e. embedded in another
 * structure).
 *
 * Return: %0 on success, negative error code otherwise.
 */
int mempool_init_noprof(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
                        mempool_free_t *free_fn, void *pool_data)
{
        return mempool_init_node(pool, min_nr, alloc_fn, free_fn,
                                 pool_data, GFP_KERNEL, NUMA_NO_NODE);

}
EXPORT_SYMBOL(mempool_init_noprof);

/**
 * mempool_create_node - create a memory pool
 * @min_nr:    the minimum number of elements guaranteed to be
 *             allocated for this pool.
 * @alloc_fn:  user-defined element-allocation function.
 * @free_fn:   user-defined element-freeing function.
 * @pool_data: optional private data available to the user-defined functions.
 * @gfp_mask:  memory allocation flags
 * @node_id:   numa node to allocate on
 *
 * this function creates and allocates a guaranteed size, preallocated
 * memory pool. The pool can be used from the mempool_alloc() and mempool_free()
 * functions. This function might sleep. Both the alloc_fn() and the free_fn()
 * functions might sleep - as long as the mempool_alloc() function is not called
 * from IRQ contexts.
 *
 * Return: pointer to the created memory pool object or %NULL on error.
 */
mempool_t *mempool_create_node_noprof(int min_nr, mempool_alloc_t *alloc_fn,
                                      mempool_free_t *free_fn, void *pool_data,
                                      gfp_t gfp_mask, int node_id)
{
        mempool_t *pool;

        pool = kmalloc_node_noprof(sizeof(*pool), gfp_mask | __GFP_ZERO, node_id);
        if (!pool)
                return NULL;

        if (mempool_init_node(pool, min_nr, alloc_fn, free_fn, pool_data,
                              gfp_mask, node_id)) {
                kfree(pool);
                return NULL;
        }

        return pool;
}
EXPORT_SYMBOL(mempool_create_node_noprof);

/**
 * mempool_resize - resize an existing memory pool
 * @pool:       pointer to the memory pool which was allocated via
 *              mempool_create().
 * @new_min_nr: the new minimum number of elements guaranteed to be
 *              allocated for this pool.
 *
 * This function shrinks/grows the pool. In the case of growing,
 * it cannot be guaranteed that the pool will be grown to the new
 * size immediately, but new mempool_free() calls will refill it.
 * This function may sleep.
 *
 * Note, the caller must guarantee that no mempool_destroy is called
 * while this function is running. mempool_alloc() & mempool_free()
 * might be called (eg. from IRQ contexts) while this function executes.
 *
 * Return: %0 on success, negative error code otherwise.
 */
int mempool_resize(mempool_t *pool, int new_min_nr)
{
        void *element;
        void **new_elements;
        unsigned long flags;

        BUG_ON(new_min_nr <= 0);
        might_sleep();

        spin_lock_irqsave(&pool->lock, flags);
        if (new_min_nr <= pool->min_nr) {
                while (new_min_nr < pool->curr_nr) {
                        element = remove_element(pool);
                        spin_unlock_irqrestore(&pool->lock, flags);
                        pool->free(element, pool->pool_data);
                        spin_lock_irqsave(&pool->lock, flags);
                }
                pool->min_nr = new_min_nr;
                goto out_unlock;
        }
        spin_unlock_irqrestore(&pool->lock, flags);

        /* Grow the pool */
        new_elements = kmalloc_array(new_min_nr, sizeof(*new_elements),
                                     GFP_KERNEL);
        if (!new_elements)
                return -ENOMEM;

        spin_lock_irqsave(&pool->lock, flags);
        if (unlikely(new_min_nr <= pool->min_nr)) {
                /* Raced, other resize will do our work */
                spin_unlock_irqrestore(&pool->lock, flags);
                kfree(new_elements);
                goto out;
        }
        memcpy(new_elements, pool->elements,
                        pool->curr_nr * sizeof(*new_elements));
        kfree(pool->elements);
        pool->elements = new_elements;
        pool->min_nr = new_min_nr;

        while (pool->curr_nr < pool->min_nr) {
                spin_unlock_irqrestore(&pool->lock, flags);
                element = pool->alloc(GFP_KERNEL, pool->pool_data);
                if (!element)
                        goto out;
                spin_lock_irqsave(&pool->lock, flags);
                if (pool->curr_nr < pool->min_nr) {
                        add_element(pool, element);
                } else {
                        spin_unlock_irqrestore(&pool->lock, flags);
                        pool->free(element, pool->pool_data);        /* Raced */
                        goto out;
                }
        }
out_unlock:
        spin_unlock_irqrestore(&pool->lock, flags);
out:
        return 0;
}
EXPORT_SYMBOL(mempool_resize);

/**
 * mempool_alloc - allocate an element from a specific memory pool
 * @pool:      pointer to the memory pool which was allocated via
 *             mempool_create().
 * @gfp_mask:  the usual allocation bitmask.
 *
 * this function only sleeps if the alloc_fn() function sleeps or
 * returns NULL. Note that due to preallocation, this function
 * *never* fails when called from process contexts. (it might
 * fail if called from an IRQ context.)
 * Note: using __GFP_ZERO is not supported.
 *
 * Return: pointer to the allocated element or %NULL on error.
 */
void *mempool_alloc_noprof(mempool_t *pool, gfp_t gfp_mask)
{
        void *element;
        unsigned long flags;
        wait_queue_entry_t wait;
        gfp_t gfp_temp;

        VM_WARN_ON_ONCE(gfp_mask & __GFP_ZERO);
        might_alloc(gfp_mask);

        gfp_mask |= __GFP_NOMEMALLOC;        /* don't allocate emergency reserves */
        gfp_mask |= __GFP_NORETRY;        /* don't loop in __alloc_pages */
        gfp_mask |= __GFP_NOWARN;        /* failures are OK */

        gfp_temp = gfp_mask & ~(__GFP_DIRECT_RECLAIM|__GFP_IO);

repeat_alloc:

        element = pool->alloc(gfp_temp, pool->pool_data);
        if (likely(element != NULL))
                return element;

        spin_lock_irqsave(&pool->lock, flags);
        if (likely(pool->curr_nr)) {
                element = remove_element(pool);
                spin_unlock_irqrestore(&pool->lock, flags);
                /* paired with rmb in mempool_free(), read comment there */
                smp_wmb();
                /*
                 * Update the allocation stack trace as this is more useful
                 * for debugging.
                 */
                kmemleak_update_trace(element);
                return element;
        }

        /*
         * We use gfp mask w/o direct reclaim or IO for the first round.  If
         * alloc failed with that and @pool was empty, retry immediately.
         */
        if (gfp_temp != gfp_mask) {
                spin_unlock_irqrestore(&pool->lock, flags);
                gfp_temp = gfp_mask;
                goto repeat_alloc;
        }

        /* We must not sleep if !__GFP_DIRECT_RECLAIM */
        if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) {
                spin_unlock_irqrestore(&pool->lock, flags);
                return NULL;
        }

        /* Let's wait for someone else to return an element to @pool */
        init_wait(&wait);
        prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);

        spin_unlock_irqrestore(&pool->lock, flags);

        /*
         * FIXME: this should be io_schedule().  The timeout is there as a
         * workaround for some DM problems in 2.6.18.
         */
        io_schedule_timeout(5*HZ);

        finish_wait(&pool->wait, &wait);
        goto repeat_alloc;
}
EXPORT_SYMBOL(mempool_alloc_noprof);

/**
 * mempool_alloc_preallocated - allocate an element from preallocated elements
 *                              belonging to a specific memory pool
 * @pool:      pointer to the memory pool which was allocated via
 *             mempool_create().
 *
 * This function is similar to mempool_alloc, but it only attempts allocating
 * an element from the preallocated elements. It does not sleep and immediately
 * returns if no preallocated elements are available.
 *
 * Return: pointer to the allocated element or %NULL if no elements are
 * available.
 */
void *mempool_alloc_preallocated(mempool_t *pool)
{
        void *element;
        unsigned long flags;

        spin_lock_irqsave(&pool->lock, flags);
        if (likely(pool->curr_nr)) {
                element = remove_element(pool);
                spin_unlock_irqrestore(&pool->lock, flags);
                /* paired with rmb in mempool_free(), read comment there */
                smp_wmb();
                /*
                 * Update the allocation stack trace as this is more useful
                 * for debugging.
                 */
                kmemleak_update_trace(element);
                return element;
        }
        spin_unlock_irqrestore(&pool->lock, flags);

        return NULL;
}
EXPORT_SYMBOL(mempool_alloc_preallocated);

/**
 * mempool_free - return an element to the pool.
 * @element:   pool element pointer.
 * @pool:      pointer to the memory pool which was allocated via
 *             mempool_create().
 *
 * this function only sleeps if the free_fn() function sleeps.
 */
void mempool_free(void *element, mempool_t *pool)
{
        unsigned long flags;

        if (unlikely(element == NULL))
                return;

        /*
         * Paired with the wmb in mempool_alloc().  The preceding read is
         * for @element and the following @pool->curr_nr.  This ensures
         * that the visible value of @pool->curr_nr is from after the
         * allocation of @element.  This is necessary for fringe cases
         * where @element was passed to this task without going through
         * barriers.
         *
         * For example, assume @p is %NULL at the beginning and one task
         * performs "p = mempool_alloc(...);" while another task is doing
         * "while (!p) cpu_relax(); mempool_free(p, ...);".  This function
         * may end up using curr_nr value which is from before allocation
         * of @p without the following rmb.
         */
        smp_rmb();

        /*
         * For correctness, we need a test which is guaranteed to trigger
         * if curr_nr + #allocated == min_nr.  Testing curr_nr < min_nr
         * without locking achieves that and refilling as soon as possible
         * is desirable.
         *
         * Because curr_nr visible here is always a value after the
         * allocation of @element, any task which decremented curr_nr below
         * min_nr is guaranteed to see curr_nr < min_nr unless curr_nr gets
         * incremented to min_nr afterwards.  If curr_nr gets incremented
         * to min_nr after the allocation of @element, the elements
         * allocated after that are subject to the same guarantee.
         *
         * Waiters happen iff curr_nr is 0 and the above guarantee also
         * ensures that there will be frees which return elements to the
         * pool waking up the waiters.
         */
        if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) {
                spin_lock_irqsave(&pool->lock, flags);
                if (likely(pool->curr_nr < pool->min_nr)) {
                        add_element(pool, element);
                        spin_unlock_irqrestore(&pool->lock, flags);
                        wake_up(&pool->wait);
                        return;
                }
                spin_unlock_irqrestore(&pool->lock, flags);
        }
        pool->free(element, pool->pool_data);
}
EXPORT_SYMBOL(mempool_free);

/*
 * A commonly used alloc and free fn.
 */
void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data)
{
        struct kmem_cache *mem = pool_data;
        VM_BUG_ON(mem->ctor);
        return kmem_cache_alloc_noprof(mem, gfp_mask);
}
EXPORT_SYMBOL(mempool_alloc_slab);

void mempool_free_slab(void *element, void *pool_data)
{
        struct kmem_cache *mem = pool_data;
        kmem_cache_free(mem, element);
}
EXPORT_SYMBOL(mempool_free_slab);

/*
 * A commonly used alloc and free fn that kmalloc/kfrees the amount of memory
 * specified by pool_data
 */
void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data)
{
        size_t size = (size_t)pool_data;
        return kmalloc_noprof(size, gfp_mask);
}
EXPORT_SYMBOL(mempool_kmalloc);

void mempool_kfree(void *element, void *pool_data)
{
        kfree(element);
}
EXPORT_SYMBOL(mempool_kfree);

void *mempool_kvmalloc(gfp_t gfp_mask, void *pool_data)
{
        size_t size = (size_t)pool_data;
        return kvmalloc(size, gfp_mask);
}
EXPORT_SYMBOL(mempool_kvmalloc);

void mempool_kvfree(void *element, void *pool_data)
{
        kvfree(element);
}
EXPORT_SYMBOL(mempool_kvfree);

/*
 * A simple mempool-backed page allocator that allocates pages
 * of the order specified by pool_data.
 */
void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data)
{
        int order = (int)(long)pool_data;
        return alloc_pages_noprof(gfp_mask, order);
}
EXPORT_SYMBOL(mempool_alloc_pages);

void mempool_free_pages(void *element, void *pool_data)
{
        int order = (int)(long)pool_data;
        __free_pages(element, order);
}
EXPORT_SYMBOL(mempool_free_pages);


































































  290 



































  292 





























  293 










  292 





















  293 






  292 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
/*
 * kmod - the kernel module loader
 *
 * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org>
 */

#include <linux/module.h>
#include <linux/sched.h>
#include <linux/sched/task.h>
#include <linux/binfmts.h>
#include <linux/syscalls.h>
#include <linux/unistd.h>
#include <linux/kmod.h>
#include <linux/slab.h>
#include <linux/completion.h>
#include <linux/cred.h>
#include <linux/file.h>
#include <linux/workqueue.h>
#include <linux/security.h>
#include <linux/mount.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/resource.h>
#include <linux/notifier.h>
#include <linux/suspend.h>
#include <linux/rwsem.h>
#include <linux/ptrace.h>
#include <linux/async.h>
#include <linux/uaccess.h>

#include <trace/events/module.h>
#include "internal.h"

/*
 * Assuming:
 *
 * threads = div64_u64((u64) totalram_pages * (u64) PAGE_SIZE,
 *                       (u64) THREAD_SIZE * 8UL);
 *
 * If you need less than 50 threads would mean we're dealing with systems
 * smaller than 3200 pages. This assumes you are capable of having ~13M memory,
 * and this would only be an upper limit, after which the OOM killer would take
 * effect. Systems like these are very unlikely if modules are enabled.
 */
#define MAX_KMOD_CONCURRENT 50
static DEFINE_SEMAPHORE(kmod_concurrent_max, MAX_KMOD_CONCURRENT);

/*
 * This is a restriction on having *all* MAX_KMOD_CONCURRENT threads
 * running at the same time without returning. When this happens we
 * believe you've somehow ended up with a recursive module dependency
 * creating a loop.
 *
 * We have no option but to fail.
 *
 * Userspace should proactively try to detect and prevent these.
 */
#define MAX_KMOD_ALL_BUSY_TIMEOUT 5

/*
        modprobe_path is set via /proc/sys.
*/
char modprobe_path[KMOD_PATH_LEN] = CONFIG_MODPROBE_PATH;

static void free_modprobe_argv(struct subprocess_info *info)
{
        kfree(info->argv[3]); /* check call_modprobe() */
        kfree(info->argv);
}

static int call_modprobe(char *orig_module_name, int wait)
{
        struct subprocess_info *info;
        static char *envp[] = {
                "HOME=/",
                "TERM=linux",
                "PATH=/sbin:/usr/sbin:/bin:/usr/bin",
                NULL
        };
        char *module_name;
        int ret;

        char **argv = kmalloc(sizeof(char *[5]), GFP_KERNEL);
        if (!argv)
                goto out;

        module_name = kstrdup(orig_module_name, GFP_KERNEL);
        if (!module_name)
                goto free_argv;

        argv[0] = modprobe_path;
        argv[1] = "-q";
        argv[2] = "--";
        argv[3] = module_name;        /* check free_modprobe_argv() */
        argv[4] = NULL;

        info = call_usermodehelper_setup(modprobe_path, argv, envp, GFP_KERNEL,
                                         NULL, free_modprobe_argv, NULL);
        if (!info)
                goto free_module_name;

        ret = call_usermodehelper_exec(info, wait | UMH_KILLABLE);
        kmod_dup_request_announce(orig_module_name, ret);
        return ret;

free_module_name:
        kfree(module_name);
free_argv:
        kfree(argv);
out:
        kmod_dup_request_announce(orig_module_name, -ENOMEM);
        return -ENOMEM;
}

/**
 * __request_module - try to load a kernel module
 * @wait: wait (or not) for the operation to complete
 * @fmt: printf style format string for the name of the module
 * @...: arguments as specified in the format string
 *
 * Load a module using the user mode module loader. The function returns
 * zero on success or a negative errno code or positive exit code from
 * "modprobe" on failure. Note that a successful module load does not mean
 * the module did not then unload and exit on an error of its own. Callers
 * must check that the service they requested is now available not blindly
 * invoke it.
 *
 * If module auto-loading support is disabled then this function
 * simply returns -ENOENT.
 */
int __request_module(bool wait, const char *fmt, ...)
{
        va_list args;
        char module_name[MODULE_NAME_LEN];
        int ret, dup_ret;

        /*
         * We don't allow synchronous module loading from async.  Module
         * init may invoke async_synchronize_full() which will end up
         * waiting for this task which already is waiting for the module
         * loading to complete, leading to a deadlock.
         */
        WARN_ON_ONCE(wait && current_is_async());

        if (!modprobe_path[0])
                return -ENOENT;

        va_start(args, fmt);
        ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
        va_end(args);
        if (ret >= MODULE_NAME_LEN)
                return -ENAMETOOLONG;

        ret = security_kernel_module_request(module_name);
        if (ret)
                return ret;

        ret = down_timeout(&kmod_concurrent_max, MAX_KMOD_ALL_BUSY_TIMEOUT * HZ);
        if (ret) {
                pr_warn_ratelimited("request_module: modprobe %s cannot be processed, kmod busy with %d threads for more than %d seconds now",
                                    module_name, MAX_KMOD_CONCURRENT, MAX_KMOD_ALL_BUSY_TIMEOUT);
                return ret;
        }

        trace_module_request(module_name, wait, _RET_IP_);

        if (kmod_dup_request_exists_wait(module_name, wait, &dup_ret)) {
                ret = dup_ret;
                goto out;
        }

        ret = call_modprobe(module_name, wait ? UMH_WAIT_PROC : UMH_WAIT_EXEC);

out:
        up(&kmod_concurrent_max);

        return ret;
}
EXPORT_SYMBOL(__request_module);
















































































































































































































 5855 















































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_JUMP_LABEL_H
#define _LINUX_JUMP_LABEL_H

/*
 * Jump label support
 *
 * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
 * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
 *
 * DEPRECATED API:
 *
 * The use of 'struct static_key' directly, is now DEPRECATED. In addition
 * static_key_{true,false}() is also DEPRECATED. IE DO NOT use the following:
 *
 * struct static_key false = STATIC_KEY_INIT_FALSE;
 * struct static_key true = STATIC_KEY_INIT_TRUE;
 * static_key_true()
 * static_key_false()
 *
 * The updated API replacements are:
 *
 * DEFINE_STATIC_KEY_TRUE(key);
 * DEFINE_STATIC_KEY_FALSE(key);
 * DEFINE_STATIC_KEY_ARRAY_TRUE(keys, count);
 * DEFINE_STATIC_KEY_ARRAY_FALSE(keys, count);
 * static_branch_likely()
 * static_branch_unlikely()
 *
 * Jump labels provide an interface to generate dynamic branches using
 * self-modifying code. Assuming toolchain and architecture support, if we
 * define a "key" that is initially false via "DEFINE_STATIC_KEY_FALSE(key)",
 * an "if (static_branch_unlikely(&key))" statement is an unconditional branch
 * (which defaults to false - and the true block is placed out of line).
 * Similarly, we can define an initially true key via
 * "DEFINE_STATIC_KEY_TRUE(key)", and use it in the same
 * "if (static_branch_unlikely(&key))", in which case we will generate an
 * unconditional branch to the out-of-line true branch. Keys that are
 * initially true or false can be using in both static_branch_unlikely()
 * and static_branch_likely() statements.
 *
 * At runtime we can change the branch target by setting the key
 * to true via a call to static_branch_enable(), or false using
 * static_branch_disable(). If the direction of the branch is switched by
 * these calls then we run-time modify the branch target via a
 * no-op -> jump or jump -> no-op conversion. For example, for an
 * initially false key that is used in an "if (static_branch_unlikely(&key))"
 * statement, setting the key to true requires us to patch in a jump
 * to the out-of-line of true branch.
 *
 * In addition to static_branch_{enable,disable}, we can also reference count
 * the key or branch direction via static_branch_{inc,dec}. Thus,
 * static_branch_inc() can be thought of as a 'make more true' and
 * static_branch_dec() as a 'make more false'.
 *
 * Since this relies on modifying code, the branch modifying functions
 * must be considered absolute slow paths (machine wide synchronization etc.).
 * OTOH, since the affected branches are unconditional, their runtime overhead
 * will be absolutely minimal, esp. in the default (off) case where the total
 * effect is a single NOP of appropriate size. The on case will patch in a jump
 * to the out-of-line block.
 *
 * When the control is directly exposed to userspace, it is prudent to delay the
 * decrement to avoid high frequency code modifications which can (and do)
 * cause significant performance degradation. Struct static_key_deferred and
 * static_key_slow_dec_deferred() provide for this.
 *
 * Lacking toolchain and or architecture support, static keys fall back to a
 * simple conditional branch.
 *
 * Additional babbling in: Documentation/staging/static-keys.rst
 */

#ifndef __ASSEMBLY__

#include <linux/types.h>
#include <linux/compiler.h>
#include <linux/cleanup.h>

extern bool static_key_initialized;

#define STATIC_KEY_CHECK_USE(key) WARN(!static_key_initialized,                      \
                                    "%s(): static key '%pS' used before call to jump_label_init()", \
                                    __func__, (key))

struct static_key {
        atomic_t enabled;
#ifdef CONFIG_JUMP_LABEL
/*
 * Note:
 *   To make anonymous unions work with old compilers, the static
 *   initialization of them requires brackets. This creates a dependency
 *   on the order of the struct with the initializers. If any fields
 *   are added, STATIC_KEY_INIT_TRUE and STATIC_KEY_INIT_FALSE may need
 *   to be modified.
 *
 * bit 0 => 1 if key is initially true
 *            0 if initially false
 * bit 1 => 1 if points to struct static_key_mod
 *            0 if points to struct jump_entry
 */
        union {
                unsigned long type;
                struct jump_entry *entries;
                struct static_key_mod *next;
        };
#endif        /* CONFIG_JUMP_LABEL */
};

#endif /* __ASSEMBLY__ */

#ifdef CONFIG_JUMP_LABEL
#include <asm/jump_label.h>

#ifndef __ASSEMBLY__
#ifdef CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE

struct jump_entry {
        s32 code;
        s32 target;
        long key;        // key may be far away from the core kernel under KASLR
};

static inline unsigned long jump_entry_code(const struct jump_entry *entry)
{
        return (unsigned long)&entry->code + entry->code;
}

static inline unsigned long jump_entry_target(const struct jump_entry *entry)
{
        return (unsigned long)&entry->target + entry->target;
}

static inline struct static_key *jump_entry_key(const struct jump_entry *entry)
{
        long offset = entry->key & ~3L;

        return (struct static_key *)((unsigned long)&entry->key + offset);
}

#else

static inline unsigned long jump_entry_code(const struct jump_entry *entry)
{
        return entry->code;
}

static inline unsigned long jump_entry_target(const struct jump_entry *entry)
{
        return entry->target;
}

static inline struct static_key *jump_entry_key(const struct jump_entry *entry)
{
        return (struct static_key *)((unsigned long)entry->key & ~3UL);
}

#endif

static inline bool jump_entry_is_branch(const struct jump_entry *entry)
{
        return (unsigned long)entry->key & 1UL;
}

static inline bool jump_entry_is_init(const struct jump_entry *entry)
{
        return (unsigned long)entry->key & 2UL;
}

static inline void jump_entry_set_init(struct jump_entry *entry, bool set)
{
        if (set)
                entry->key |= 2;
        else
                entry->key &= ~2;
}

static inline int jump_entry_size(struct jump_entry *entry)
{
#ifdef JUMP_LABEL_NOP_SIZE
        return JUMP_LABEL_NOP_SIZE;
#else
        return arch_jump_entry_size(entry);
#endif
}

#endif
#endif

#ifndef __ASSEMBLY__

enum jump_label_type {
        JUMP_LABEL_NOP = 0,
        JUMP_LABEL_JMP,
};

struct module;

#ifdef CONFIG_JUMP_LABEL

#define JUMP_TYPE_FALSE                0UL
#define JUMP_TYPE_TRUE                1UL
#define JUMP_TYPE_LINKED        2UL
#define JUMP_TYPE_MASK                3UL

static __always_inline bool static_key_false(struct static_key *key)
{
        return arch_static_branch(key, false);
}

static __always_inline bool static_key_true(struct static_key *key)
{
        return !arch_static_branch(key, true);
}

extern struct jump_entry __start___jump_table[];
extern struct jump_entry __stop___jump_table[];

extern void jump_label_init(void);
extern void jump_label_init_ro(void);
extern void jump_label_lock(void);
extern void jump_label_unlock(void);
extern void arch_jump_label_transform(struct jump_entry *entry,
                                      enum jump_label_type type);
extern bool arch_jump_label_transform_queue(struct jump_entry *entry,
                                            enum jump_label_type type);
extern void arch_jump_label_transform_apply(void);
extern int jump_label_text_reserved(void *start, void *end);
extern bool static_key_slow_inc(struct static_key *key);
extern bool static_key_fast_inc_not_disabled(struct static_key *key);
extern void static_key_slow_dec(struct static_key *key);
extern bool static_key_slow_inc_cpuslocked(struct static_key *key);
extern void static_key_slow_dec_cpuslocked(struct static_key *key);
extern int static_key_count(struct static_key *key);
extern void static_key_enable(struct static_key *key);
extern void static_key_disable(struct static_key *key);
extern void static_key_enable_cpuslocked(struct static_key *key);
extern void static_key_disable_cpuslocked(struct static_key *key);
extern enum jump_label_type jump_label_init_type(struct jump_entry *entry);

/*
 * We should be using ATOMIC_INIT() for initializing .enabled, but
 * the inclusion of atomic.h is problematic for inclusion of jump_label.h
 * in 'low-level' headers. Thus, we are initializing .enabled with a
 * raw value, but have added a BUILD_BUG_ON() to catch any issues in
 * jump_label_init() see: kernel/jump_label.c.
 */
#define STATIC_KEY_INIT_TRUE                                        \
        { .enabled = { 1 },                                        \
          { .type = JUMP_TYPE_TRUE } }
#define STATIC_KEY_INIT_FALSE                                        \
        { .enabled = { 0 },                                        \
          { .type = JUMP_TYPE_FALSE } }

#else  /* !CONFIG_JUMP_LABEL */

#include <linux/atomic.h>
#include <linux/bug.h>

static __always_inline int static_key_count(struct static_key *key)
{
        return raw_atomic_read(&key->enabled);
}

static __always_inline void jump_label_init(void)
{
        static_key_initialized = true;
}

static __always_inline void jump_label_init_ro(void) { }

static __always_inline bool static_key_false(struct static_key *key)
{
        if (unlikely_notrace(static_key_count(key) > 0))
                return true;
        return false;
}

static __always_inline bool static_key_true(struct static_key *key)
{
        if (likely_notrace(static_key_count(key) > 0))
                return true;
        return false;
}

static inline bool static_key_fast_inc_not_disabled(struct static_key *key)
{
        int v;

        STATIC_KEY_CHECK_USE(key);
        /*
         * Prevent key->enabled getting negative to follow the same semantics
         * as for CONFIG_JUMP_LABEL=y, see kernel/jump_label.c comment.
         */
        v = atomic_read(&key->enabled);
        do {
                if (v < 0 || (v + 1) < 0)
                        return false;
        } while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v + 1)));
        return true;
}
#define static_key_slow_inc(key)        static_key_fast_inc_not_disabled(key)

static inline void static_key_slow_dec(struct static_key *key)
{
        STATIC_KEY_CHECK_USE(key);
        atomic_dec(&key->enabled);
}

#define static_key_slow_inc_cpuslocked(key) static_key_slow_inc(key)
#define static_key_slow_dec_cpuslocked(key) static_key_slow_dec(key)

static inline int jump_label_text_reserved(void *start, void *end)
{
        return 0;
}

static inline void jump_label_lock(void) {}
static inline void jump_label_unlock(void) {}

static inline void static_key_enable(struct static_key *key)
{
        STATIC_KEY_CHECK_USE(key);

        if (atomic_read(&key->enabled) != 0) {
                WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
                return;
        }
        atomic_set(&key->enabled, 1);
}

static inline void static_key_disable(struct static_key *key)
{
        STATIC_KEY_CHECK_USE(key);

        if (atomic_read(&key->enabled) != 1) {
                WARN_ON_ONCE(atomic_read(&key->enabled) != 0);
                return;
        }
        atomic_set(&key->enabled, 0);
}

#define static_key_enable_cpuslocked(k)                static_key_enable((k))
#define static_key_disable_cpuslocked(k)        static_key_disable((k))

#define STATIC_KEY_INIT_TRUE        { .enabled = ATOMIC_INIT(1) }
#define STATIC_KEY_INIT_FALSE        { .enabled = ATOMIC_INIT(0) }

#endif        /* CONFIG_JUMP_LABEL */

DEFINE_LOCK_GUARD_0(jump_label_lock, jump_label_lock(), jump_label_unlock())

#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
#define jump_label_enabled static_key_enabled

/* -------------------------------------------------------------------------- */

/*
 * Two type wrappers around static_key, such that we can use compile time
 * type differentiation to emit the right code.
 *
 * All the below code is macros in order to play type games.
 */

struct static_key_true {
        struct static_key key;
};

struct static_key_false {
        struct static_key key;
};

#define STATIC_KEY_TRUE_INIT  (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE,  }
#define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, }

#define DEFINE_STATIC_KEY_TRUE(name)        \
        struct static_key_true name = STATIC_KEY_TRUE_INIT

#define DEFINE_STATIC_KEY_TRUE_RO(name)        \
        struct static_key_true name __ro_after_init = STATIC_KEY_TRUE_INIT

#define DECLARE_STATIC_KEY_TRUE(name)        \
        extern struct static_key_true name

#define DEFINE_STATIC_KEY_FALSE(name)        \
        struct static_key_false name = STATIC_KEY_FALSE_INIT

#define DEFINE_STATIC_KEY_FALSE_RO(name)        \
        struct static_key_false name __ro_after_init = STATIC_KEY_FALSE_INIT

#define DECLARE_STATIC_KEY_FALSE(name)        \
        extern struct static_key_false name

#define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count)                \
        struct static_key_true name[count] = {                        \
                [0 ... (count) - 1] = STATIC_KEY_TRUE_INIT,        \
        }

#define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count)                \
        struct static_key_false name[count] = {                        \
                [0 ... (count) - 1] = STATIC_KEY_FALSE_INIT,        \
        }

#define _DEFINE_STATIC_KEY_1(name)        DEFINE_STATIC_KEY_TRUE(name)
#define _DEFINE_STATIC_KEY_0(name)        DEFINE_STATIC_KEY_FALSE(name)
#define DEFINE_STATIC_KEY_MAYBE(cfg, name)                        \
        __PASTE(_DEFINE_STATIC_KEY_, IS_ENABLED(cfg))(name)

#define _DEFINE_STATIC_KEY_RO_1(name)        DEFINE_STATIC_KEY_TRUE_RO(name)
#define _DEFINE_STATIC_KEY_RO_0(name)        DEFINE_STATIC_KEY_FALSE_RO(name)
#define DEFINE_STATIC_KEY_MAYBE_RO(cfg, name)                        \
        __PASTE(_DEFINE_STATIC_KEY_RO_, IS_ENABLED(cfg))(name)

#define _DECLARE_STATIC_KEY_1(name)        DECLARE_STATIC_KEY_TRUE(name)
#define _DECLARE_STATIC_KEY_0(name)        DECLARE_STATIC_KEY_FALSE(name)
#define DECLARE_STATIC_KEY_MAYBE(cfg, name)                        \
        __PASTE(_DECLARE_STATIC_KEY_, IS_ENABLED(cfg))(name)

extern bool ____wrong_branch_error(void);

#define static_key_enabled(x)                                                        \
({                                                                                \
        if (!__builtin_types_compatible_p(typeof(*x), struct static_key) &&        \
            !__builtin_types_compatible_p(typeof(*x), struct static_key_true) &&\
            !__builtin_types_compatible_p(typeof(*x), struct static_key_false))        \
                ____wrong_branch_error();                                        \
        static_key_count((struct static_key *)x) > 0;                                \
})

#ifdef CONFIG_JUMP_LABEL

/*
 * Combine the right initial value (type) with the right branch order
 * to generate the desired result.
 *
 *
 * type\branch|        likely (1)              |        unlikely (0)
 * -----------+-----------------------+------------------
 *            |                       |
 *  true (1)  |           ...                      |           ...
 *            |    NOP                      |           JMP L
 *            |    <br-stmts>              |        1: ...
 *            |        L: ...                      |
 *            |                              |
 *            |                              |        L: <br-stmts>
 *            |                              |           jmp 1b
 *            |                       |
 * -----------+-----------------------+------------------
 *            |                       |
 *  false (0) |           ...                      |           ...
 *            |    JMP L              |           NOP
 *            |    <br-stmts>              |        1: ...
 *            |        L: ...                      |
 *            |                              |
 *            |                              |        L: <br-stmts>
 *            |                              |           jmp 1b
 *            |                       |
 * -----------+-----------------------+------------------
 *
 * The initial value is encoded in the LSB of static_key::entries,
 * type: 0 = false, 1 = true.
 *
 * The branch type is encoded in the LSB of jump_entry::key,
 * branch: 0 = unlikely, 1 = likely.
 *
 * This gives the following logic table:
 *
 *        enabled        type        branch          instuction
 * -----------------------------+-----------
 *        0        0        0        | NOP
 *        0        0        1        | JMP
 *        0        1        0        | NOP
 *        0        1        1        | JMP
 *
 *        1        0        0        | JMP
 *        1        0        1        | NOP
 *        1        1        0        | JMP
 *        1        1        1        | NOP
 *
 * Which gives the following functions:
 *
 *   dynamic: instruction = enabled ^ branch
 *   static:  instruction = type ^ branch
 *
 * See jump_label_type() / jump_label_init_type().
 */

#define static_branch_likely(x)                                                        \
({                                                                                \
        bool branch;                                                                \
        if (__builtin_types_compatible_p(typeof(*x), struct static_key_true))        \
                branch = !arch_static_branch(&(x)->key, true);                        \
        else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
                branch = !arch_static_branch_jump(&(x)->key, true);                \
        else                                                                        \
                branch = ____wrong_branch_error();                                \
        likely_notrace(branch);                                                                \
})

#define static_branch_unlikely(x)                                                \
({                                                                                \
        bool branch;                                                                \
        if (__builtin_types_compatible_p(typeof(*x), struct static_key_true))        \
                branch = arch_static_branch_jump(&(x)->key, false);                \
        else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
                branch = arch_static_branch(&(x)->key, false);                        \
        else                                                                        \
                branch = ____wrong_branch_error();                                \
        unlikely_notrace(branch);                                                        \
})

#else /* !CONFIG_JUMP_LABEL */

#define static_branch_likely(x)                likely_notrace(static_key_enabled(&(x)->key))
#define static_branch_unlikely(x)        unlikely_notrace(static_key_enabled(&(x)->key))

#endif /* CONFIG_JUMP_LABEL */

#define static_branch_maybe(config, x)                                        \
        (IS_ENABLED(config) ? static_branch_likely(x)                        \
                            : static_branch_unlikely(x))

/*
 * Advanced usage; refcount, branch is enabled when: count != 0
 */

#define static_branch_inc(x)                static_key_slow_inc(&(x)->key)
#define static_branch_dec(x)                static_key_slow_dec(&(x)->key)
#define static_branch_inc_cpuslocked(x)        static_key_slow_inc_cpuslocked(&(x)->key)
#define static_branch_dec_cpuslocked(x)        static_key_slow_dec_cpuslocked(&(x)->key)

/*
 * Normal usage; boolean enable/disable.
 */

#define static_branch_enable(x)                        static_key_enable(&(x)->key)
#define static_branch_disable(x)                static_key_disable(&(x)->key)
#define static_branch_enable_cpuslocked(x)        static_key_enable_cpuslocked(&(x)->key)
#define static_branch_disable_cpuslocked(x)        static_key_disable_cpuslocked(&(x)->key)

#endif /* __ASSEMBLY__ */

#endif        /* _LINUX_JUMP_LABEL_H */

















































































































































   30 
   30 
   30 








































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
// SPDX-License-Identifier: GPL-2.0
/*
 * Central processing for nfsd.
 *
 * Authors:        Olaf Kirch (okir@monad.swb.de)
 *
 * Copyright (C) 1995, 1996, 1997 Olaf Kirch <okir@monad.swb.de>
 */

#include <linux/sched/signal.h>
#include <linux/freezer.h>
#include <linux/module.h>
#include <linux/fs_struct.h>
#include <linux/swap.h>
#include <linux/siphash.h>

#include <linux/sunrpc/stats.h>
#include <linux/sunrpc/svcsock.h>
#include <linux/sunrpc/svc_xprt.h>
#include <linux/lockd/bind.h>
#include <linux/nfsacl.h>
#include <linux/nfslocalio.h>
#include <linux/seq_file.h>
#include <linux/inetdevice.h>
#include <net/addrconf.h>
#include <net/ipv6.h>
#include <net/net_namespace.h>
#include "nfsd.h"
#include "cache.h"
#include "vfs.h"
#include "netns.h"
#include "filecache.h"

#include "trace.h"

#define NFSDDBG_FACILITY        NFSDDBG_SVC

atomic_t                        nfsd_th_cnt = ATOMIC_INIT(0);
static int                        nfsd(void *vrqstp);
#if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL)
static int                        nfsd_acl_rpcbind_set(struct net *,
                                                     const struct svc_program *,
                                                     u32, int,
                                                     unsigned short,
                                                     unsigned short);
static __be32                        nfsd_acl_init_request(struct svc_rqst *,
                                                const struct svc_program *,
                                                struct svc_process_info *);
#endif
static int                        nfsd_rpcbind_set(struct net *,
                                                 const struct svc_program *,
                                                 u32, int,
                                                 unsigned short,
                                                 unsigned short);
static __be32                        nfsd_init_request(struct svc_rqst *,
                                                const struct svc_program *,
                                                struct svc_process_info *);

/*
 * nfsd_mutex protects nn->nfsd_serv -- both the pointer itself and some members
 * of the svc_serv struct such as ->sv_temp_socks and ->sv_permsocks.
 *
 * Finally, the nfsd_mutex also protects some of the global variables that are
 * accessed when nfsd starts and that are settable via the write_* routines in
 * nfsctl.c. In particular:
 *
 *        user_recovery_dirname
 *        user_lease_time
 *        nfsd_versions
 */
DEFINE_MUTEX(nfsd_mutex);

#if IS_ENABLED(CONFIG_NFS_LOCALIO)
static const struct svc_version *localio_versions[] = {
        [1] = &localio_version1,
};

#define NFSD_LOCALIO_NRVERS                ARRAY_SIZE(localio_versions)

#endif /* CONFIG_NFS_LOCALIO */

#if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL)
static const struct svc_version *nfsd_acl_version[] = {
# if defined(CONFIG_NFSD_V2_ACL)
        [2] = &nfsd_acl_version2,
# endif
# if defined(CONFIG_NFSD_V3_ACL)
        [3] = &nfsd_acl_version3,
# endif
};

#define NFSD_ACL_MINVERS        2
#define NFSD_ACL_NRVERS                ARRAY_SIZE(nfsd_acl_version)

#endif /* defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) */

static const struct svc_version *nfsd_version[NFSD_MAXVERS+1] = {
#if defined(CONFIG_NFSD_V2)
        [2] = &nfsd_version2,
#endif
        [3] = &nfsd_version3,
#if defined(CONFIG_NFSD_V4)
        [4] = &nfsd_version4,
#endif
};

struct svc_program                nfsd_programs[] = {
        {
        .pg_prog                = NFS_PROGRAM,                /* program number */
        .pg_nvers                = NFSD_MAXVERS+1,        /* nr of entries in nfsd_version */
        .pg_vers                = nfsd_version,                /* version table */
        .pg_name                = "nfsd",                /* program name */
        .pg_class                = "nfsd",                /* authentication class */
        .pg_authenticate        = svc_set_client,        /* export authentication */
        .pg_init_request        = nfsd_init_request,
        .pg_rpcbind_set                = nfsd_rpcbind_set,
        },
#if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL)
        {
        .pg_prog                = NFS_ACL_PROGRAM,
        .pg_nvers                = NFSD_ACL_NRVERS,
        .pg_vers                = nfsd_acl_version,
        .pg_name                = "nfsacl",
        .pg_class                = "nfsd",
        .pg_authenticate        = svc_set_client,
        .pg_init_request        = nfsd_acl_init_request,
        .pg_rpcbind_set                = nfsd_acl_rpcbind_set,
        },
#endif /* defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) */
#if IS_ENABLED(CONFIG_NFS_LOCALIO)
        {
        .pg_prog                = NFS_LOCALIO_PROGRAM,
        .pg_nvers                = NFSD_LOCALIO_NRVERS,
        .pg_vers                = localio_versions,
        .pg_name                = "nfslocalio",
        .pg_class                = "nfsd",
        .pg_authenticate        = svc_set_client,
        .pg_init_request        = svc_generic_init_request,
        .pg_rpcbind_set                = svc_generic_rpcbind_set,
        }
#endif /* CONFIG_NFS_LOCALIO */
};

bool nfsd_support_version(int vers)
{
        if (vers >= NFSD_MINVERS && vers <= NFSD_MAXVERS)
                return nfsd_version[vers] != NULL;
        return false;
}

int nfsd_vers(struct nfsd_net *nn, int vers, enum vers_op change)
{
        if (vers < NFSD_MINVERS || vers > NFSD_MAXVERS)
                return 0;
        switch(change) {
        case NFSD_SET:
                nn->nfsd_versions[vers] = nfsd_support_version(vers);
                break;
        case NFSD_CLEAR:
                nn->nfsd_versions[vers] = false;
                break;
        case NFSD_TEST:
                return nn->nfsd_versions[vers];
        case NFSD_AVAIL:
                return nfsd_support_version(vers);
        }
        return 0;
}

static void
nfsd_adjust_nfsd_versions4(struct nfsd_net *nn)
{
        unsigned i;

        for (i = 0; i <= NFSD_SUPPORTED_MINOR_VERSION; i++) {
                if (nn->nfsd4_minorversions[i])
                        return;
        }
        nfsd_vers(nn, 4, NFSD_CLEAR);
}

int nfsd_minorversion(struct nfsd_net *nn, u32 minorversion, enum vers_op change)
{
        if (minorversion > NFSD_SUPPORTED_MINOR_VERSION &&
            change != NFSD_AVAIL)
                return -1;

        switch(change) {
        case NFSD_SET:
                nfsd_vers(nn, 4, NFSD_SET);
                nn->nfsd4_minorversions[minorversion] =
                        nfsd_vers(nn, 4, NFSD_TEST);
                break;
        case NFSD_CLEAR:
                nn->nfsd4_minorversions[minorversion] = false;
                nfsd_adjust_nfsd_versions4(nn);
                break;
        case NFSD_TEST:
                return nn->nfsd4_minorversions[minorversion];
        case NFSD_AVAIL:
                return minorversion <= NFSD_SUPPORTED_MINOR_VERSION &&
                        nfsd_vers(nn, 4, NFSD_AVAIL);
        }
        return 0;
}

bool nfsd_net_try_get(struct net *net) __must_hold(rcu)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        return (nn && percpu_ref_tryget_live(&nn->nfsd_net_ref));
}

void nfsd_net_put(struct net *net) __must_hold(rcu)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        percpu_ref_put(&nn->nfsd_net_ref);
}

static void nfsd_net_done(struct percpu_ref *ref)
{
        struct nfsd_net *nn = container_of(ref, struct nfsd_net, nfsd_net_ref);

        complete(&nn->nfsd_net_confirm_done);
}

static void nfsd_net_free(struct percpu_ref *ref)
{
        struct nfsd_net *nn = container_of(ref, struct nfsd_net, nfsd_net_ref);

        complete(&nn->nfsd_net_free_done);
}

/*
 * Maximum number of nfsd processes
 */
#define        NFSD_MAXSERVS                8192

int nfsd_nrthreads(struct net *net)
{
        int rv = 0;
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        mutex_lock(&nfsd_mutex);
        if (nn->nfsd_serv)
                rv = nn->nfsd_serv->sv_nrthreads;
        mutex_unlock(&nfsd_mutex);
        return rv;
}

static int nfsd_init_socks(struct net *net, const struct cred *cred)
{
        int error;
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        if (!list_empty(&nn->nfsd_serv->sv_permsocks))
                return 0;

        error = svc_xprt_create(nn->nfsd_serv, "udp", net, PF_INET, NFS_PORT,
                                SVC_SOCK_DEFAULTS, cred);
        if (error < 0)
                return error;

        error = svc_xprt_create(nn->nfsd_serv, "tcp", net, PF_INET, NFS_PORT,
                                SVC_SOCK_DEFAULTS, cred);
        if (error < 0)
                return error;

        return 0;
}

static int nfsd_users = 0;

static int nfsd_startup_generic(void)
{
        int ret;

        if (nfsd_users++)
                return 0;

        ret = nfsd_file_cache_init();
        if (ret)
                goto dec_users;

        ret = nfs4_state_start();
        if (ret)
                goto out_file_cache;
        return 0;

out_file_cache:
        nfsd_file_cache_shutdown();
dec_users:
        nfsd_users--;
        return ret;
}

static void nfsd_shutdown_generic(void)
{
        if (--nfsd_users)
                return;

        nfs4_state_shutdown();
        nfsd_file_cache_shutdown();
}

static bool nfsd_needs_lockd(struct nfsd_net *nn)
{
        return nfsd_vers(nn, 2, NFSD_TEST) || nfsd_vers(nn, 3, NFSD_TEST);
}

/**
 * nfsd_copy_write_verifier - Atomically copy a write verifier
 * @verf: buffer in which to receive the verifier cookie
 * @nn: NFS net namespace
 *
 * This function provides a wait-free mechanism for copying the
 * namespace's write verifier without tearing it.
 */
void nfsd_copy_write_verifier(__be32 verf[2], struct nfsd_net *nn)
{
        unsigned int seq;

        do {
                seq = read_seqbegin(&nn->writeverf_lock);
                memcpy(verf, nn->writeverf, sizeof(nn->writeverf));
        } while (read_seqretry(&nn->writeverf_lock, seq));
}

static void nfsd_reset_write_verifier_locked(struct nfsd_net *nn)
{
        struct timespec64 now;
        u64 verf;

        /*
         * Because the time value is hashed, y2038 time_t overflow
         * is irrelevant in this usage.
         */
        ktime_get_raw_ts64(&now);
        verf = siphash_2u64(now.tv_sec, now.tv_nsec, &nn->siphash_key);
        memcpy(nn->writeverf, &verf, sizeof(nn->writeverf));
}

/**
 * nfsd_reset_write_verifier - Generate a new write verifier
 * @nn: NFS net namespace
 *
 * This function updates the ->writeverf field of @nn. This field
 * contains an opaque cookie that, according to Section 18.32.3 of
 * RFC 8881, "the client can use to determine whether a server has
 * changed instance state (e.g., server restart) between a call to
 * WRITE and a subsequent call to either WRITE or COMMIT.  This
 * cookie MUST be unchanged during a single instance of the NFSv4.1
 * server and MUST be unique between instances of the NFSv4.1
 * server."
 */
void nfsd_reset_write_verifier(struct nfsd_net *nn)
{
        write_seqlock(&nn->writeverf_lock);
        nfsd_reset_write_verifier_locked(nn);
        write_sequnlock(&nn->writeverf_lock);
}

/*
 * Crank up a set of per-namespace resources for a new NFSD instance,
 * including lockd, a duplicate reply cache, an open file cache
 * instance, and a cache of NFSv4 state objects.
 */
static int nfsd_startup_net(struct net *net, const struct cred *cred)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        int ret;

        if (nn->nfsd_net_up)
                return 0;

        ret = nfsd_startup_generic();
        if (ret)
                return ret;
        ret = nfsd_init_socks(net, cred);
        if (ret)
                goto out_socks;

        if (nfsd_needs_lockd(nn) && !nn->lockd_up) {
                ret = lockd_up(net, cred);
                if (ret)
                        goto out_socks;
                nn->lockd_up = true;
        }

        ret = nfsd_file_cache_start_net(net);
        if (ret)
                goto out_lockd;

        ret = nfsd_reply_cache_init(nn);
        if (ret)
                goto out_filecache;

        ret = nfs4_state_start_net(net);
        if (ret)
                goto out_reply_cache;

#ifdef CONFIG_NFSD_V4_2_INTER_SSC
        nfsd4_ssc_init_umount_work(nn);
#endif
        nn->nfsd_net_up = true;
        return 0;

out_reply_cache:
        nfsd_reply_cache_shutdown(nn);
out_filecache:
        nfsd_file_cache_shutdown_net(net);
out_lockd:
        if (nn->lockd_up) {
                lockd_down(net);
                nn->lockd_up = false;
        }
out_socks:
        nfsd_shutdown_generic();
        return ret;
}

static void nfsd_shutdown_net(struct net *net)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        if (!nn->nfsd_net_up)
                return;

        percpu_ref_kill_and_confirm(&nn->nfsd_net_ref, nfsd_net_done);
        wait_for_completion(&nn->nfsd_net_confirm_done);

        nfsd_export_flush(net);
        nfs4_state_shutdown_net(net);
        nfsd_reply_cache_shutdown(nn);
        nfsd_file_cache_shutdown_net(net);
        if (nn->lockd_up) {
                lockd_down(net);
                nn->lockd_up = false;
        }

        wait_for_completion(&nn->nfsd_net_free_done);
        percpu_ref_exit(&nn->nfsd_net_ref);

        nn->nfsd_net_up = false;
        nfsd_shutdown_generic();
}

static DEFINE_SPINLOCK(nfsd_notifier_lock);
static int nfsd_inetaddr_event(struct notifier_block *this, unsigned long event,
        void *ptr)
{
        struct in_ifaddr *ifa = (struct in_ifaddr *)ptr;
        struct net_device *dev = ifa->ifa_dev->dev;
        struct net *net = dev_net(dev);
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        struct sockaddr_in sin;

        if (event != NETDEV_DOWN || !nn->nfsd_serv)
                goto out;

        spin_lock(&nfsd_notifier_lock);
        if (nn->nfsd_serv) {
                dprintk("nfsd_inetaddr_event: removed %pI4\n", &ifa->ifa_local);
                sin.sin_family = AF_INET;
                sin.sin_addr.s_addr = ifa->ifa_local;
                svc_age_temp_xprts_now(nn->nfsd_serv, (struct sockaddr *)&sin);
        }
        spin_unlock(&nfsd_notifier_lock);

out:
        return NOTIFY_DONE;
}

static struct notifier_block nfsd_inetaddr_notifier = {
        .notifier_call = nfsd_inetaddr_event,
};

#if IS_ENABLED(CONFIG_IPV6)
static int nfsd_inet6addr_event(struct notifier_block *this,
        unsigned long event, void *ptr)
{
        struct inet6_ifaddr *ifa = (struct inet6_ifaddr *)ptr;
        struct net_device *dev = ifa->idev->dev;
        struct net *net = dev_net(dev);
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        struct sockaddr_in6 sin6;

        if (event != NETDEV_DOWN || !nn->nfsd_serv)
                goto out;

        spin_lock(&nfsd_notifier_lock);
        if (nn->nfsd_serv) {
                dprintk("nfsd_inet6addr_event: removed %pI6\n", &ifa->addr);
                sin6.sin6_family = AF_INET6;
                sin6.sin6_addr = ifa->addr;
                if (ipv6_addr_type(&sin6.sin6_addr) & IPV6_ADDR_LINKLOCAL)
                        sin6.sin6_scope_id = ifa->idev->dev->ifindex;
                svc_age_temp_xprts_now(nn->nfsd_serv, (struct sockaddr *)&sin6);
        }
        spin_unlock(&nfsd_notifier_lock);

out:
        return NOTIFY_DONE;
}

static struct notifier_block nfsd_inet6addr_notifier = {
        .notifier_call = nfsd_inet6addr_event,
};
#endif

/* Only used under nfsd_mutex, so this atomic may be overkill: */
static atomic_t nfsd_notifier_refcount = ATOMIC_INIT(0);

/**
 * nfsd_destroy_serv - tear down NFSD's svc_serv for a namespace
 * @net: network namespace the NFS service is associated with
 */
void nfsd_destroy_serv(struct net *net)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        struct svc_serv *serv = nn->nfsd_serv;

        lockdep_assert_held(&nfsd_mutex);

        spin_lock(&nfsd_notifier_lock);
        nn->nfsd_serv = NULL;
        spin_unlock(&nfsd_notifier_lock);

        /* check if the notifier still has clients */
        if (atomic_dec_return(&nfsd_notifier_refcount) == 0) {
                unregister_inetaddr_notifier(&nfsd_inetaddr_notifier);
#if IS_ENABLED(CONFIG_IPV6)
                unregister_inet6addr_notifier(&nfsd_inet6addr_notifier);
#endif
        }

        svc_xprt_destroy_all(serv, net);

        /*
         * write_ports can create the server without actually starting
         * any threads--if we get shut down before any threads are
         * started, then nfsd_destroy_serv will be run before any of this
         * other initialization has been done except the rpcb information.
         */
        svc_rpcb_cleanup(serv, net);

        nfsd_shutdown_net(net);
        svc_destroy(&serv);
}

void nfsd_reset_versions(struct nfsd_net *nn)
{
        int i;

        for (i = 0; i <= NFSD_MAXVERS; i++)
                if (nfsd_vers(nn, i, NFSD_TEST))
                        return;

        for (i = 0; i <= NFSD_MAXVERS; i++)
                if (i != 4)
                        nfsd_vers(nn, i, NFSD_SET);
                else {
                        int minor = 0;
                        while (nfsd_minorversion(nn, minor, NFSD_SET) >= 0)
                                minor++;
                }
}

static int nfsd_get_default_max_blksize(void)
{
        struct sysinfo i;
        unsigned long long target;
        unsigned long ret;

        si_meminfo(&i);
        target = (i.totalram - i.totalhigh) << PAGE_SHIFT;
        /*
         * Aim for 1/4096 of memory per thread This gives 1MB on 4Gig
         * machines, but only uses 32K on 128M machines.  Bottom out at
         * 8K on 32M and smaller.  Of course, this is only a default.
         */
        target >>= 12;

        ret = NFSSVC_MAXBLKSIZE;
        while (ret > target && ret >= 8*1024*2)
                ret /= 2;
        return ret;
}

void nfsd_shutdown_threads(struct net *net)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        struct svc_serv *serv;

        mutex_lock(&nfsd_mutex);
        serv = nn->nfsd_serv;
        if (serv == NULL) {
                mutex_unlock(&nfsd_mutex);
                return;
        }

        /* Kill outstanding nfsd threads */
        svc_set_num_threads(serv, NULL, 0);
        nfsd_destroy_serv(net);
        mutex_unlock(&nfsd_mutex);
}

struct svc_rqst *nfsd_current_rqst(void)
{
        if (kthread_func(current) == nfsd)
                return kthread_data(current);
        return NULL;
}

int nfsd_create_serv(struct net *net)
{
        int error;
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        struct svc_serv *serv;

        WARN_ON(!mutex_is_locked(&nfsd_mutex));
        if (nn->nfsd_serv)
                return 0;

        error = percpu_ref_init(&nn->nfsd_net_ref, nfsd_net_free,
                                0, GFP_KERNEL);
        if (error)
                return error;
        init_completion(&nn->nfsd_net_free_done);
        init_completion(&nn->nfsd_net_confirm_done);

        if (nfsd_max_blksize == 0)
                nfsd_max_blksize = nfsd_get_default_max_blksize();
        nfsd_reset_versions(nn);
        serv = svc_create_pooled(nfsd_programs, ARRAY_SIZE(nfsd_programs),
                                 &nn->nfsd_svcstats,
                                 nfsd_max_blksize, nfsd);
        if (serv == NULL)
                return -ENOMEM;

        error = svc_bind(serv, net);
        if (error < 0) {
                svc_destroy(&serv);
                return error;
        }
        spin_lock(&nfsd_notifier_lock);
        nn->nfsd_serv = serv;
        spin_unlock(&nfsd_notifier_lock);

        /* check if the notifier is already set */
        if (atomic_inc_return(&nfsd_notifier_refcount) == 1) {
                register_inetaddr_notifier(&nfsd_inetaddr_notifier);
#if IS_ENABLED(CONFIG_IPV6)
                register_inet6addr_notifier(&nfsd_inet6addr_notifier);
#endif
        }
        nfsd_reset_write_verifier(nn);
        return 0;
}

int nfsd_nrpools(struct net *net)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        if (nn->nfsd_serv == NULL)
                return 0;
        else
                return nn->nfsd_serv->sv_nrpools;
}

int nfsd_get_nrthreads(int n, int *nthreads, struct net *net)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        struct svc_serv *serv = nn->nfsd_serv;
        int i;

        if (serv)
                for (i = 0; i < serv->sv_nrpools && i < n; i++)
                        nthreads[i] = serv->sv_pools[i].sp_nrthreads;
        return 0;
}

/**
 * nfsd_set_nrthreads - set the number of running threads in the net's service
 * @n: number of array members in @nthreads
 * @nthreads: array of thread counts for each pool
 * @net: network namespace to operate within
 *
 * This function alters the number of running threads for the given network
 * namespace in each pool. If passed an array longer then the number of pools
 * the extra pool settings are ignored. If passed an array shorter than the
 * number of pools, the missing values are interpreted as 0's.
 *
 * Returns 0 on success or a negative errno on error.
 */
int nfsd_set_nrthreads(int n, int *nthreads, struct net *net)
{
        int i = 0;
        int tot = 0;
        int err = 0;
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        lockdep_assert_held(&nfsd_mutex);

        if (nn->nfsd_serv == NULL || n <= 0)
                return 0;

        /*
         * Special case: When n == 1, pass in NULL for the pool, so that the
         * change is distributed equally among them.
         */
        if (n == 1)
                return svc_set_num_threads(nn->nfsd_serv, NULL, nthreads[0]);

        if (n > nn->nfsd_serv->sv_nrpools)
                n = nn->nfsd_serv->sv_nrpools;

        /* enforce a global maximum number of threads */
        tot = 0;
        for (i = 0; i < n; i++) {
                nthreads[i] = min(nthreads[i], NFSD_MAXSERVS);
                tot += nthreads[i];
        }
        if (tot > NFSD_MAXSERVS) {
                /* total too large: scale down requested numbers */
                for (i = 0; i < n && tot > 0; i++) {
                        int new = nthreads[i] * NFSD_MAXSERVS / tot;
                        tot -= (nthreads[i] - new);
                        nthreads[i] = new;
                }
                for (i = 0; i < n && tot > 0; i++) {
                        nthreads[i]--;
                        tot--;
                }
        }

        /* apply the new numbers */
        for (i = 0; i < n; i++) {
                err = svc_set_num_threads(nn->nfsd_serv,
                                          &nn->nfsd_serv->sv_pools[i],
                                          nthreads[i]);
                if (err)
                        goto out;
        }

        /* Anything undefined in array is considered to be 0 */
        for (i = n; i < nn->nfsd_serv->sv_nrpools; ++i) {
                err = svc_set_num_threads(nn->nfsd_serv,
                                          &nn->nfsd_serv->sv_pools[i],
                                          0);
                if (err)
                        goto out;
        }
out:
        return err;
}

/**
 * nfsd_svc: start up or shut down the nfsd server
 * @n: number of array members in @nthreads
 * @nthreads: array of thread counts for each pool
 * @net: network namespace to operate within
 * @cred: credentials to use for xprt creation
 * @scope: server scope value (defaults to nodename)
 *
 * Adjust the number of threads in each pool and return the new
 * total number of threads in the service.
 */
int
nfsd_svc(int n, int *nthreads, struct net *net, const struct cred *cred, const char *scope)
{
        int        error;
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        struct svc_serv *serv;

        lockdep_assert_held(&nfsd_mutex);

        dprintk("nfsd: creating service\n");

        strscpy(nn->nfsd_name, scope ? scope : utsname()->nodename,
                sizeof(nn->nfsd_name));

        error = nfsd_create_serv(net);
        if (error)
                goto out;
        serv = nn->nfsd_serv;

        error = nfsd_startup_net(net, cred);
        if (error)
                goto out_put;
        error = nfsd_set_nrthreads(n, nthreads, net);
        if (error)
                goto out_put;
        error = serv->sv_nrthreads;
out_put:
        if (serv->sv_nrthreads == 0)
                nfsd_destroy_serv(net);
out:
        return error;
}

#if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL)
static bool
nfsd_support_acl_version(int vers)
{
        if (vers >= NFSD_ACL_MINVERS && vers < NFSD_ACL_NRVERS)
                return nfsd_acl_version[vers] != NULL;
        return false;
}

static int
nfsd_acl_rpcbind_set(struct net *net, const struct svc_program *progp,
                     u32 version, int family, unsigned short proto,
                     unsigned short port)
{
        if (!nfsd_support_acl_version(version) ||
            !nfsd_vers(net_generic(net, nfsd_net_id), version, NFSD_TEST))
                return 0;
        return svc_generic_rpcbind_set(net, progp, version, family,
                        proto, port);
}

static __be32
nfsd_acl_init_request(struct svc_rqst *rqstp,
                      const struct svc_program *progp,
                      struct svc_process_info *ret)
{
        struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
        int i;

        if (likely(nfsd_support_acl_version(rqstp->rq_vers) &&
            nfsd_vers(nn, rqstp->rq_vers, NFSD_TEST)))
                return svc_generic_init_request(rqstp, progp, ret);

        ret->mismatch.lovers = NFSD_ACL_NRVERS;
        for (i = NFSD_ACL_MINVERS; i < NFSD_ACL_NRVERS; i++) {
                if (nfsd_support_acl_version(rqstp->rq_vers) &&
                    nfsd_vers(nn, i, NFSD_TEST)) {
                        ret->mismatch.lovers = i;
                        break;
                }
        }
        if (ret->mismatch.lovers == NFSD_ACL_NRVERS)
                return rpc_prog_unavail;
        ret->mismatch.hivers = NFSD_ACL_MINVERS;
        for (i = NFSD_ACL_NRVERS - 1; i >= NFSD_ACL_MINVERS; i--) {
                if (nfsd_support_acl_version(rqstp->rq_vers) &&
                    nfsd_vers(nn, i, NFSD_TEST)) {
                        ret->mismatch.hivers = i;
                        break;
                }
        }
        return rpc_prog_mismatch;
}
#endif

static int
nfsd_rpcbind_set(struct net *net, const struct svc_program *progp,
                 u32 version, int family, unsigned short proto,
                 unsigned short port)
{
        if (!nfsd_vers(net_generic(net, nfsd_net_id), version, NFSD_TEST))
                return 0;
        return svc_generic_rpcbind_set(net, progp, version, family,
                        proto, port);
}

static __be32
nfsd_init_request(struct svc_rqst *rqstp,
                  const struct svc_program *progp,
                  struct svc_process_info *ret)
{
        struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
        int i;

        if (likely(nfsd_vers(nn, rqstp->rq_vers, NFSD_TEST)))
                return svc_generic_init_request(rqstp, progp, ret);

        ret->mismatch.lovers = NFSD_MAXVERS + 1;
        for (i = NFSD_MINVERS; i <= NFSD_MAXVERS; i++) {
                if (nfsd_vers(nn, i, NFSD_TEST)) {
                        ret->mismatch.lovers = i;
                        break;
                }
        }
        if (ret->mismatch.lovers > NFSD_MAXVERS)
                return rpc_prog_unavail;
        ret->mismatch.hivers = NFSD_MINVERS;
        for (i = NFSD_MAXVERS; i >= NFSD_MINVERS; i--) {
                if (nfsd_vers(nn, i, NFSD_TEST)) {
                        ret->mismatch.hivers = i;
                        break;
                }
        }
        return rpc_prog_mismatch;
}

/*
 * This is the NFS server kernel thread
 */
static int
nfsd(void *vrqstp)
{
        struct svc_rqst *rqstp = (struct svc_rqst *) vrqstp;
        struct svc_xprt *perm_sock = list_entry(rqstp->rq_server->sv_permsocks.next, typeof(struct svc_xprt), xpt_list);
        struct net *net = perm_sock->xpt_net;
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        /* At this point, the thread shares current->fs
         * with the init process. We need to create files with the
         * umask as defined by the client instead of init's umask.
         */
        svc_thread_init_status(rqstp, unshare_fs_struct());

        current->fs->umask = 0;

        atomic_inc(&nfsd_th_cnt);

        set_freezable();

        /*
         * The main request loop
         */
        while (!svc_thread_should_stop(rqstp)) {
                svc_recv(rqstp);
                nfsd_file_net_dispose(nn);
        }

        atomic_dec(&nfsd_th_cnt);

        /* Release the thread */
        svc_exit_thread(rqstp);
        return 0;
}

/**
 * nfsd_dispatch - Process an NFS or NFSACL or LOCALIO Request
 * @rqstp: incoming request
 *
 * This RPC dispatcher integrates the NFS server's duplicate reply cache.
 *
 * Return values:
 *  %0: Processing complete; do not send a Reply
 *  %1: Processing complete; send Reply in rqstp->rq_res
 */
int nfsd_dispatch(struct svc_rqst *rqstp)
{
        const struct svc_procedure *proc = rqstp->rq_procinfo;
        __be32 *statp = rqstp->rq_accept_statp;
        struct nfsd_cacherep *rp;
        unsigned int start, len;
        __be32 *nfs_reply;

        /*
         * Give the xdr decoder a chance to change this if it wants
         * (necessary in the NFSv4.0 compound case)
         */
        rqstp->rq_cachetype = proc->pc_cachetype;

        /*
         * ->pc_decode advances the argument stream past the NFS
         * Call header, so grab the header's starting location and
         * size now for the call to nfsd_cache_lookup().
         */
        start = xdr_stream_pos(&rqstp->rq_arg_stream);
        len = xdr_stream_remaining(&rqstp->rq_arg_stream);
        if (!proc->pc_decode(rqstp, &rqstp->rq_arg_stream))
                goto out_decode_err;

        /*
         * Release rq_status_counter setting it to an odd value after the rpc
         * request has been properly parsed. rq_status_counter is used to
         * notify the consumers if the rqstp fields are stable
         * (rq_status_counter is odd) or not meaningful (rq_status_counter
         * is even).
         */
        smp_store_release(&rqstp->rq_status_counter, rqstp->rq_status_counter | 1);

        rp = NULL;
        switch (nfsd_cache_lookup(rqstp, start, len, &rp)) {
        case RC_DOIT:
                break;
        case RC_REPLY:
                goto out_cached_reply;
        case RC_DROPIT:
                goto out_dropit;
        }

        nfs_reply = xdr_inline_decode(&rqstp->rq_res_stream, 0);
        *statp = proc->pc_func(rqstp);
        if (test_bit(RQ_DROPME, &rqstp->rq_flags))
                goto out_update_drop;

        if (!proc->pc_encode(rqstp, &rqstp->rq_res_stream))
                goto out_encode_err;

        /*
         * Release rq_status_counter setting it to an even value after the rpc
         * request has been properly processed.
         */
        smp_store_release(&rqstp->rq_status_counter, rqstp->rq_status_counter + 1);

        nfsd_cache_update(rqstp, rp, rqstp->rq_cachetype, nfs_reply);
out_cached_reply:
        return 1;

out_decode_err:
        trace_nfsd_garbage_args_err(rqstp);
        *statp = rpc_garbage_args;
        return 1;

out_update_drop:
        nfsd_cache_update(rqstp, rp, RC_NOCACHE, NULL);
out_dropit:
        return 0;

out_encode_err:
        trace_nfsd_cant_encode_err(rqstp);
        nfsd_cache_update(rqstp, rp, RC_NOCACHE, NULL);
        *statp = rpc_system_err;
        return 1;
}

/**
 * nfssvc_decode_voidarg - Decode void arguments
 * @rqstp: Server RPC transaction context
 * @xdr: XDR stream positioned at arguments to decode
 *
 * Return values:
 *   %false: Arguments were not valid
 *   %true: Decoding was successful
 */
bool nfssvc_decode_voidarg(struct svc_rqst *rqstp, struct xdr_stream *xdr)
{
        return true;
}

/**
 * nfssvc_encode_voidres - Encode void results
 * @rqstp: Server RPC transaction context
 * @xdr: XDR stream into which to encode results
 *
 * Return values:
 *   %false: Local error while encoding
 *   %true: Encoding was successful
 */
bool nfssvc_encode_voidres(struct svc_rqst *rqstp, struct xdr_stream *xdr)
{
        return true;
}




































































































































































































































































































































































   10 





   79 

   79 









  113 





















   10 





  112 









  113 


  113 










































   10 





   10 

   10 


































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
// SPDX-License-Identifier: GPL-2.0
/*
 *  linux/fs/char_dev.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 */

#include <linux/init.h>
#include <linux/fs.h>
#include <linux/kdev_t.h>
#include <linux/slab.h>
#include <linux/string.h>

#include <linux/major.h>
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/seq_file.h>

#include <linux/kobject.h>
#include <linux/kobj_map.h>
#include <linux/cdev.h>
#include <linux/mutex.h>
#include <linux/backing-dev.h>
#include <linux/tty.h>

#include "internal.h"

static struct kobj_map *cdev_map __ro_after_init;

static DEFINE_MUTEX(chrdevs_lock);

#define CHRDEV_MAJOR_HASH_SIZE 255

static struct char_device_struct {
        struct char_device_struct *next;
        unsigned int major;
        unsigned int baseminor;
        int minorct;
        char name[64];
        struct cdev *cdev;                /* will die */
} *chrdevs[CHRDEV_MAJOR_HASH_SIZE];

/* index in the above */
static inline int major_to_index(unsigned major)
{
        return major % CHRDEV_MAJOR_HASH_SIZE;
}

#ifdef CONFIG_PROC_FS

void chrdev_show(struct seq_file *f, off_t offset)
{
        struct char_device_struct *cd;

        mutex_lock(&chrdevs_lock);
        for (cd = chrdevs[major_to_index(offset)]; cd; cd = cd->next) {
                if (cd->major == offset)
                        seq_printf(f, "%3d %s\n", cd->major, cd->name);
        }
        mutex_unlock(&chrdevs_lock);
}

#endif /* CONFIG_PROC_FS */

static int find_dynamic_major(void)
{
        int i;
        struct char_device_struct *cd;

        for (i = ARRAY_SIZE(chrdevs)-1; i >= CHRDEV_MAJOR_DYN_END; i--) {
                if (chrdevs[i] == NULL)
                        return i;
        }

        for (i = CHRDEV_MAJOR_DYN_EXT_START;
             i >= CHRDEV_MAJOR_DYN_EXT_END; i--) {
                for (cd = chrdevs[major_to_index(i)]; cd; cd = cd->next)
                        if (cd->major == i)
                                break;

                if (cd == NULL)
                        return i;
        }

        return -EBUSY;
}

/*
 * Register a single major with a specified minor range.
 *
 * If major == 0 this function will dynamically allocate an unused major.
 * If major > 0 this function will attempt to reserve the range of minors
 * with given major.
 *
 */
static struct char_device_struct *
__register_chrdev_region(unsigned int major, unsigned int baseminor,
                           int minorct, const char *name)
{
        struct char_device_struct *cd, *curr, *prev = NULL;
        int ret;
        int i;

        if (major >= CHRDEV_MAJOR_MAX) {
                pr_err("CHRDEV \"%s\" major requested (%u) is greater than the maximum (%u)\n",
                       name, major, CHRDEV_MAJOR_MAX-1);
                return ERR_PTR(-EINVAL);
        }

        if (minorct > MINORMASK + 1 - baseminor) {
                pr_err("CHRDEV \"%s\" minor range requested (%u-%u) is out of range of maximum range (%u-%u) for a single major\n",
                        name, baseminor, baseminor + minorct - 1, 0, MINORMASK);
                return ERR_PTR(-EINVAL);
        }

        cd = kzalloc(sizeof(struct char_device_struct), GFP_KERNEL);
        if (cd == NULL)
                return ERR_PTR(-ENOMEM);

        mutex_lock(&chrdevs_lock);

        if (major == 0) {
                ret = find_dynamic_major();
                if (ret < 0) {
                        pr_err("CHRDEV \"%s\" dynamic allocation region is full\n",
                               name);
                        goto out;
                }
                major = ret;
        }

        ret = -EBUSY;
        i = major_to_index(major);
        for (curr = chrdevs[i]; curr; prev = curr, curr = curr->next) {
                if (curr->major < major)
                        continue;

                if (curr->major > major)
                        break;

                if (curr->baseminor + curr->minorct <= baseminor)
                        continue;

                if (curr->baseminor >= baseminor + minorct)
                        break;

                goto out;
        }

        cd->major = major;
        cd->baseminor = baseminor;
        cd->minorct = minorct;
        strscpy(cd->name, name, sizeof(cd->name));

        if (!prev) {
                cd->next = curr;
                chrdevs[i] = cd;
        } else {
                cd->next = prev->next;
                prev->next = cd;
        }

        mutex_unlock(&chrdevs_lock);
        return cd;
out:
        mutex_unlock(&chrdevs_lock);
        kfree(cd);
        return ERR_PTR(ret);
}

static struct char_device_struct *
__unregister_chrdev_region(unsigned major, unsigned baseminor, int minorct)
{
        struct char_device_struct *cd = NULL, **cp;
        int i = major_to_index(major);

        mutex_lock(&chrdevs_lock);
        for (cp = &chrdevs[i]; *cp; cp = &(*cp)->next)
                if ((*cp)->major == major &&
                    (*cp)->baseminor == baseminor &&
                    (*cp)->minorct == minorct)
                        break;
        if (*cp) {
                cd = *cp;
                *cp = cd->next;
        }
        mutex_unlock(&chrdevs_lock);
        return cd;
}

/**
 * register_chrdev_region() - register a range of device numbers
 * @from: the first in the desired range of device numbers; must include
 *        the major number.
 * @count: the number of consecutive device numbers required
 * @name: the name of the device or driver.
 *
 * Return value is zero on success, a negative error code on failure.
 */
int register_chrdev_region(dev_t from, unsigned count, const char *name)
{
        struct char_device_struct *cd;
        dev_t to = from + count;
        dev_t n, next;

        for (n = from; n < to; n = next) {
                next = MKDEV(MAJOR(n)+1, 0);
                if (next > to)
                        next = to;
                cd = __register_chrdev_region(MAJOR(n), MINOR(n),
                               next - n, name);
                if (IS_ERR(cd))
                        goto fail;
        }
        return 0;
fail:
        to = n;
        for (n = from; n < to; n = next) {
                next = MKDEV(MAJOR(n)+1, 0);
                kfree(__unregister_chrdev_region(MAJOR(n), MINOR(n), next - n));
        }
        return PTR_ERR(cd);
}

/**
 * alloc_chrdev_region() - register a range of char device numbers
 * @dev: output parameter for first assigned number
 * @baseminor: first of the requested range of minor numbers
 * @count: the number of minor numbers required
 * @name: the name of the associated device or driver
 *
 * Allocates a range of char device numbers.  The major number will be
 * chosen dynamically, and returned (along with the first minor number)
 * in @dev.  Returns zero or a negative error code.
 */
int alloc_chrdev_region(dev_t *dev, unsigned baseminor, unsigned count,
                        const char *name)
{
        struct char_device_struct *cd;
        cd = __register_chrdev_region(0, baseminor, count, name);
        if (IS_ERR(cd))
                return PTR_ERR(cd);
        *dev = MKDEV(cd->major, cd->baseminor);
        return 0;
}

/**
 * __register_chrdev() - create and register a cdev occupying a range of minors
 * @major: major device number or 0 for dynamic allocation
 * @baseminor: first of the requested range of minor numbers
 * @count: the number of minor numbers required
 * @name: name of this range of devices
 * @fops: file operations associated with this devices
 *
 * If @major == 0 this functions will dynamically allocate a major and return
 * its number.
 *
 * If @major > 0 this function will attempt to reserve a device with the given
 * major number and will return zero on success.
 *
 * Returns a -ve errno on failure.
 *
 * The name of this device has nothing to do with the name of the device in
 * /dev. It only helps to keep track of the different owners of devices. If
 * your module name has only one type of devices it's ok to use e.g. the name
 * of the module here.
 */
int __register_chrdev(unsigned int major, unsigned int baseminor,
                      unsigned int count, const char *name,
                      const struct file_operations *fops)
{
        struct char_device_struct *cd;
        struct cdev *cdev;
        int err = -ENOMEM;

        cd = __register_chrdev_region(major, baseminor, count, name);
        if (IS_ERR(cd))
                return PTR_ERR(cd);

        cdev = cdev_alloc();
        if (!cdev)
                goto out2;

        cdev->owner = fops->owner;
        cdev->ops = fops;
        kobject_set_name(&cdev->kobj, "%s", name);

        err = cdev_add(cdev, MKDEV(cd->major, baseminor), count);
        if (err)
                goto out;

        cd->cdev = cdev;

        return major ? 0 : cd->major;
out:
        kobject_put(&cdev->kobj);
out2:
        kfree(__unregister_chrdev_region(cd->major, baseminor, count));
        return err;
}

/**
 * unregister_chrdev_region() - unregister a range of device numbers
 * @from: the first in the range of numbers to unregister
 * @count: the number of device numbers to unregister
 *
 * This function will unregister a range of @count device numbers,
 * starting with @from.  The caller should normally be the one who
 * allocated those numbers in the first place...
 */
void unregister_chrdev_region(dev_t from, unsigned count)
{
        dev_t to = from + count;
        dev_t n, next;

        for (n = from; n < to; n = next) {
                next = MKDEV(MAJOR(n)+1, 0);
                if (next > to)
                        next = to;
                kfree(__unregister_chrdev_region(MAJOR(n), MINOR(n), next - n));
        }
}

/**
 * __unregister_chrdev - unregister and destroy a cdev
 * @major: major device number
 * @baseminor: first of the range of minor numbers
 * @count: the number of minor numbers this cdev is occupying
 * @name: name of this range of devices
 *
 * Unregister and destroy the cdev occupying the region described by
 * @major, @baseminor and @count.  This function undoes what
 * __register_chrdev() did.
 */
void __unregister_chrdev(unsigned int major, unsigned int baseminor,
                         unsigned int count, const char *name)
{
        struct char_device_struct *cd;

        cd = __unregister_chrdev_region(major, baseminor, count);
        if (cd && cd->cdev)
                cdev_del(cd->cdev);
        kfree(cd);
}

static DEFINE_SPINLOCK(cdev_lock);

static struct kobject *cdev_get(struct cdev *p)
{
        struct module *owner = p->owner;
        struct kobject *kobj;

        if (!try_module_get(owner))
                return NULL;
        kobj = kobject_get_unless_zero(&p->kobj);
        if (!kobj)
                module_put(owner);
        return kobj;
}

void cdev_put(struct cdev *p)
{
        if (p) {
                struct module *owner = p->owner;
                kobject_put(&p->kobj);
                module_put(owner);
        }
}

/*
 * Called every time a character special file is opened
 */
static int chrdev_open(struct inode *inode, struct file *filp)
{
        const struct file_operations *fops;
        struct cdev *p;
        struct cdev *new = NULL;
        int ret = 0;

        spin_lock(&cdev_lock);
        p = inode->i_cdev;
        if (!p) {
                struct kobject *kobj;
                int idx;
                spin_unlock(&cdev_lock);
                kobj = kobj_lookup(cdev_map, inode->i_rdev, &idx);
                if (!kobj)
                        return -ENXIO;
                new = container_of(kobj, struct cdev, kobj);
                spin_lock(&cdev_lock);
                /* Check i_cdev again in case somebody beat us to it while
                   we dropped the lock. */
                p = inode->i_cdev;
                if (!p) {
                        inode->i_cdev = p = new;
                        list_add(&inode->i_devices, &p->list);
                        new = NULL;
                } else if (!cdev_get(p))
                        ret = -ENXIO;
        } else if (!cdev_get(p))
                ret = -ENXIO;
        spin_unlock(&cdev_lock);
        cdev_put(new);
        if (ret)
                return ret;

        ret = -ENXIO;
        fops = fops_get(p->ops);
        if (!fops)
                goto out_cdev_put;

        replace_fops(filp, fops);
        if (filp->f_op->open) {
                ret = filp->f_op->open(inode, filp);
                if (ret)
                        goto out_cdev_put;
        }

        return 0;

 out_cdev_put:
        cdev_put(p);
        return ret;
}

void cd_forget(struct inode *inode)
{
        spin_lock(&cdev_lock);
        list_del_init(&inode->i_devices);
        inode->i_cdev = NULL;
        inode->i_mapping = &inode->i_data;
        spin_unlock(&cdev_lock);
}

static void cdev_purge(struct cdev *cdev)
{
        spin_lock(&cdev_lock);
        while (!list_empty(&cdev->list)) {
                struct inode *inode;
                inode = container_of(cdev->list.next, struct inode, i_devices);
                list_del_init(&inode->i_devices);
                inode->i_cdev = NULL;
        }
        spin_unlock(&cdev_lock);
}

/*
 * Dummy default file-operations: the only thing this does
 * is contain the open that then fills in the correct operations
 * depending on the special file...
 */
const struct file_operations def_chr_fops = {
        .open = chrdev_open,
        .llseek = noop_llseek,
};

static struct kobject *exact_match(dev_t dev, int *part, void *data)
{
        struct cdev *p = data;
        return &p->kobj;
}

static int exact_lock(dev_t dev, void *data)
{
        struct cdev *p = data;
        return cdev_get(p) ? 0 : -1;
}

/**
 * cdev_add() - add a char device to the system
 * @p: the cdev structure for the device
 * @dev: the first device number for which this device is responsible
 * @count: the number of consecutive minor numbers corresponding to this
 *         device
 *
 * cdev_add() adds the device represented by @p to the system, making it
 * live immediately.  A negative error code is returned on failure.
 */
int cdev_add(struct cdev *p, dev_t dev, unsigned count)
{
        int error;

        p->dev = dev;
        p->count = count;

        if (WARN_ON(dev == WHITEOUT_DEV)) {
                error = -EBUSY;
                goto err;
        }

        error = kobj_map(cdev_map, dev, count, NULL,
                         exact_match, exact_lock, p);
        if (error)
                goto err;

        kobject_get(p->kobj.parent);

        return 0;

err:
        kfree_const(p->kobj.name);
        p->kobj.name = NULL;
        return error;
}

/**
 * cdev_set_parent() - set the parent kobject for a char device
 * @p: the cdev structure
 * @kobj: the kobject to take a reference to
 *
 * cdev_set_parent() sets a parent kobject which will be referenced
 * appropriately so the parent is not freed before the cdev. This
 * should be called before cdev_add.
 */
void cdev_set_parent(struct cdev *p, struct kobject *kobj)
{
        WARN_ON(!kobj->state_initialized);
        p->kobj.parent = kobj;
}

/**
 * cdev_device_add() - add a char device and it's corresponding
 *        struct device, linkink
 * @dev: the device structure
 * @cdev: the cdev structure
 *
 * cdev_device_add() adds the char device represented by @cdev to the system,
 * just as cdev_add does. It then adds @dev to the system using device_add
 * The dev_t for the char device will be taken from the struct device which
 * needs to be initialized first. This helper function correctly takes a
 * reference to the parent device so the parent will not get released until
 * all references to the cdev are released.
 *
 * This helper uses dev->devt for the device number. If it is not set
 * it will not add the cdev and it will be equivalent to device_add.
 *
 * This function should be used whenever the struct cdev and the
 * struct device are members of the same structure whose lifetime is
 * managed by the struct device.
 *
 * NOTE: Callers must assume that userspace was able to open the cdev and
 * can call cdev fops callbacks at any time, even if this function fails.
 */
int cdev_device_add(struct cdev *cdev, struct device *dev)
{
        int rc = 0;

        if (dev->devt) {
                cdev_set_parent(cdev, &dev->kobj);

                rc = cdev_add(cdev, dev->devt, 1);
                if (rc)
                        return rc;
        }

        rc = device_add(dev);
        if (rc && dev->devt)
                cdev_del(cdev);

        return rc;
}

/**
 * cdev_device_del() - inverse of cdev_device_add
 * @cdev: the cdev structure
 * @dev: the device structure
 *
 * cdev_device_del() is a helper function to call cdev_del and device_del.
 * It should be used whenever cdev_device_add is used.
 *
 * If dev->devt is not set it will not remove the cdev and will be equivalent
 * to device_del.
 *
 * NOTE: This guarantees that associated sysfs callbacks are not running
 * or runnable, however any cdevs already open will remain and their fops
 * will still be callable even after this function returns.
 */
void cdev_device_del(struct cdev *cdev, struct device *dev)
{
        device_del(dev);
        if (dev->devt)
                cdev_del(cdev);
}

static void cdev_unmap(dev_t dev, unsigned count)
{
        kobj_unmap(cdev_map, dev, count);
}

/**
 * cdev_del() - remove a cdev from the system
 * @p: the cdev structure to be removed
 *
 * cdev_del() removes @p from the system, possibly freeing the structure
 * itself.
 *
 * NOTE: This guarantees that cdev device will no longer be able to be
 * opened, however any cdevs already open will remain and their fops will
 * still be callable even after cdev_del returns.
 */
void cdev_del(struct cdev *p)
{
        cdev_unmap(p->dev, p->count);
        kobject_put(&p->kobj);
}


static void cdev_default_release(struct kobject *kobj)
{
        struct cdev *p = container_of(kobj, struct cdev, kobj);
        struct kobject *parent = kobj->parent;

        cdev_purge(p);
        kobject_put(parent);
}

static void cdev_dynamic_release(struct kobject *kobj)
{
        struct cdev *p = container_of(kobj, struct cdev, kobj);
        struct kobject *parent = kobj->parent;

        cdev_purge(p);
        kfree(p);
        kobject_put(parent);
}

static struct kobj_type ktype_cdev_default = {
        .release        = cdev_default_release,
};

static struct kobj_type ktype_cdev_dynamic = {
        .release        = cdev_dynamic_release,
};

/**
 * cdev_alloc() - allocate a cdev structure
 *
 * Allocates and returns a cdev structure, or NULL on failure.
 */
struct cdev *cdev_alloc(void)
{
        struct cdev *p = kzalloc(sizeof(struct cdev), GFP_KERNEL);
        if (p) {
                INIT_LIST_HEAD(&p->list);
                kobject_init(&p->kobj, &ktype_cdev_dynamic);
        }
        return p;
}

/**
 * cdev_init() - initialize a cdev structure
 * @cdev: the structure to initialize
 * @fops: the file_operations for this device
 *
 * Initializes @cdev, remembering @fops, making it ready to add to the
 * system with cdev_add().
 */
void cdev_init(struct cdev *cdev, const struct file_operations *fops)
{
        memset(cdev, 0, sizeof *cdev);
        INIT_LIST_HEAD(&cdev->list);
        kobject_init(&cdev->kobj, &ktype_cdev_default);
        cdev->ops = fops;
}

static struct kobject *base_probe(dev_t dev, int *part, void *data)
{
        if (request_module("char-major-%d-%d", MAJOR(dev), MINOR(dev)) > 0)
                /* Make old-style 2.4 aliases work */
                request_module("char-major-%d", MAJOR(dev));
        return NULL;
}

void __init chrdev_init(void)
{
        cdev_map = kobj_map_init(base_probe, &chrdevs_lock);
}


/* Let modules do char dev stuff */
EXPORT_SYMBOL(register_chrdev_region);
EXPORT_SYMBOL(unregister_chrdev_region);
EXPORT_SYMBOL(alloc_chrdev_region);
EXPORT_SYMBOL(cdev_init);
EXPORT_SYMBOL(cdev_alloc);
EXPORT_SYMBOL(cdev_del);
EXPORT_SYMBOL(cdev_add);
EXPORT_SYMBOL(cdev_set_parent);
EXPORT_SYMBOL(cdev_device_add);
EXPORT_SYMBOL(cdev_device_del);
EXPORT_SYMBOL(__register_chrdev);
EXPORT_SYMBOL(__unregister_chrdev);

















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * Resilient Queued Spin Lock defines
 *
 * (C) Copyright 2024-2025 Meta Platforms, Inc. and affiliates.
 *
 * Authors: Kumar Kartikeya Dwivedi <memxor@gmail.com>
 */
#ifndef __LINUX_RQSPINLOCK_H
#define __LINUX_RQSPINLOCK_H

#include "../locking/qspinlock.h"

/*
 * try_cmpxchg_tail - Return result of cmpxchg of tail word with a new value
 * @lock: Pointer to queued spinlock structure
 * @tail: The tail to compare against
 * @new_tail: The new queue tail code word
 * Return: Bool to indicate whether the cmpxchg operation succeeded
 *
 * This is used by the head of the wait queue to clean up the queue.
 * Provides relaxed ordering, since observers only rely on initialized
 * state of the node which was made visible through the xchg_tail operation,
 * i.e. through the smp_wmb preceding xchg_tail.
 *
 * We avoid using 16-bit cmpxchg, which is not available on all architectures.
 */
static __always_inline bool try_cmpxchg_tail(struct qspinlock *lock, u32 tail, u32 new_tail)
{
        u32 old, new;

        old = atomic_read(&lock->val);
        do {
                /*
                 * Is the tail part we compare to already stale? Fail.
                 */
                if ((old & _Q_TAIL_MASK) != tail)
                        return false;
                /*
                 * Encode latest locked/pending state for new tail.
                 */
                new = (old & _Q_LOCKED_PENDING_MASK) | new_tail;
        } while (!atomic_try_cmpxchg_relaxed(&lock->val, &old, new));

        return true;
}

#endif /* __LINUX_RQSPINLOCK_H */


































































































































































































    7 








































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_WAIT_H
#define _LINUX_WAIT_H
/*
 * Linux wait queue related types and methods
 */
#include <linux/list.h>
#include <linux/stddef.h>
#include <linux/spinlock.h>

#include <asm/current.h>

typedef struct wait_queue_entry wait_queue_entry_t;

typedef int (*wait_queue_func_t)(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
int default_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);

/* wait_queue_entry::flags */
#define WQ_FLAG_EXCLUSIVE        0x01
#define WQ_FLAG_WOKEN                0x02
#define WQ_FLAG_CUSTOM                0x04
#define WQ_FLAG_DONE                0x08
#define WQ_FLAG_PRIORITY        0x10

/*
 * A single wait-queue entry structure:
 */
struct wait_queue_entry {
        unsigned int                flags;
        void                        *private;
        wait_queue_func_t        func;
        struct list_head        entry;
};

struct wait_queue_head {
        spinlock_t                lock;
        struct list_head        head;
};
typedef struct wait_queue_head wait_queue_head_t;

struct task_struct;

/*
 * Macros for declaration and initialisaton of the datatypes
 */

#define __WAITQUEUE_INITIALIZER(name, tsk) {                                        \
        .private        = tsk,                                                        \
        .func                = default_wake_function,                                \
        .entry                = { NULL, NULL } }

#define DECLARE_WAITQUEUE(name, tsk)                                                \
        struct wait_queue_entry name = __WAITQUEUE_INITIALIZER(name, tsk)

#define __WAIT_QUEUE_HEAD_INITIALIZER(name) {                                        \
        .lock                = __SPIN_LOCK_UNLOCKED(name.lock),                        \
        .head                = LIST_HEAD_INIT(name.head) }

#define DECLARE_WAIT_QUEUE_HEAD(name) \
        struct wait_queue_head name = __WAIT_QUEUE_HEAD_INITIALIZER(name)

extern void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *);

#define init_waitqueue_head(wq_head)                                                \
        do {                                                                        \
                static struct lock_class_key __key;                                \
                                                                                \
                __init_waitqueue_head((wq_head), #wq_head, &__key);                \
        } while (0)

#ifdef CONFIG_LOCKDEP
# define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
        ({ init_waitqueue_head(&name); name; })
# define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
        struct wait_queue_head name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
#else
# define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
#endif

static inline void init_waitqueue_entry(struct wait_queue_entry *wq_entry, struct task_struct *p)
{
        wq_entry->flags                = 0;
        wq_entry->private        = p;
        wq_entry->func                = default_wake_function;
}

static inline void
init_waitqueue_func_entry(struct wait_queue_entry *wq_entry, wait_queue_func_t func)
{
        wq_entry->flags                = 0;
        wq_entry->private        = NULL;
        wq_entry->func                = func;
}

/**
 * waitqueue_active -- locklessly test for waiters on the queue
 * @wq_head: the waitqueue to test for waiters
 *
 * returns true if the wait list is not empty
 *
 * NOTE: this function is lockless and requires care, incorrect usage _will_
 * lead to sporadic and non-obvious failure.
 *
 * Use either while holding wait_queue_head::lock or when used for wakeups
 * with an extra smp_mb() like::
 *
 *      CPU0 - waker                    CPU1 - waiter
 *
 *                                      for (;;) {
 *      @cond = true;                     prepare_to_wait(&wq_head, &wait, state);
 *      smp_mb();                         // smp_mb() from set_current_state()
 *      if (waitqueue_active(wq_head))         if (@cond)
 *        wake_up(wq_head);                      break;
 *                                        schedule();
 *                                      }
 *                                      finish_wait(&wq_head, &wait);
 *
 * Because without the explicit smp_mb() it's possible for the
 * waitqueue_active() load to get hoisted over the @cond store such that we'll
 * observe an empty wait list while the waiter might not observe @cond.
 *
 * Also note that this 'optimization' trades a spin_lock() for an smp_mb(),
 * which (when the lock is uncontended) are of roughly equal cost.
 */
static inline int waitqueue_active(struct wait_queue_head *wq_head)
{
        return !list_empty(&wq_head->head);
}

/**
 * wq_has_single_sleeper - check if there is only one sleeper
 * @wq_head: wait queue head
 *
 * Returns true of wq_head has only one sleeper on the list.
 *
 * Please refer to the comment for waitqueue_active.
 */
static inline bool wq_has_single_sleeper(struct wait_queue_head *wq_head)
{
        return list_is_singular(&wq_head->head);
}

/**
 * wq_has_sleeper - check if there are any waiting processes
 * @wq_head: wait queue head
 *
 * Returns true if wq_head has waiting processes
 *
 * Please refer to the comment for waitqueue_active.
 */
static inline bool wq_has_sleeper(struct wait_queue_head *wq_head)
{
        /*
         * We need to be sure we are in sync with the
         * add_wait_queue modifications to the wait queue.
         *
         * This memory barrier should be paired with one on the
         * waiting side.
         */
        smp_mb();
        return waitqueue_active(wq_head);
}

extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
extern void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);

static inline void __add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
        struct list_head *head = &wq_head->head;
        struct wait_queue_entry *wq;

        list_for_each_entry(wq, &wq_head->head, entry) {
                if (!(wq->flags & WQ_FLAG_PRIORITY))
                        break;
                head = &wq->entry;
        }
        list_add(&wq_entry->entry, head);
}

/*
 * Used for wake-one threads:
 */
static inline void
__add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
        wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
        __add_wait_queue(wq_head, wq_entry);
}

static inline void __add_wait_queue_entry_tail(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
        list_add_tail(&wq_entry->entry, &wq_head->head);
}

static inline void
__add_wait_queue_entry_tail_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
        wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
        __add_wait_queue_entry_tail(wq_head, wq_entry);
}

static inline void
__remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
        list_del(&wq_entry->entry);
}

int __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
void __wake_up_on_current_cpu(struct wait_queue_head *wq_head, unsigned int mode, void *key);
void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
void __wake_up_locked_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr);
void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode);
void __wake_up_pollfree(struct wait_queue_head *wq_head);

#define wake_up(x)                        __wake_up(x, TASK_NORMAL, 1, NULL)
#define wake_up_nr(x, nr)                __wake_up(x, TASK_NORMAL, nr, NULL)
#define wake_up_all(x)                        __wake_up(x, TASK_NORMAL, 0, NULL)
#define wake_up_locked(x)                __wake_up_locked((x), TASK_NORMAL, 1)
#define wake_up_all_locked(x)                __wake_up_locked((x), TASK_NORMAL, 0)
#define wake_up_sync(x)                        __wake_up_sync(x, TASK_NORMAL)

#define wake_up_interruptible(x)        __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
#define wake_up_interruptible_nr(x, nr)        __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
#define wake_up_interruptible_all(x)        __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
#define wake_up_interruptible_sync(x)        __wake_up_sync((x), TASK_INTERRUPTIBLE)

/*
 * Wakeup macros to be used to report events to the targets.
 */
#define poll_to_key(m) ((void *)(__force uintptr_t)(__poll_t)(m))
#define key_to_poll(m) ((__force __poll_t)(uintptr_t)(void *)(m))
#define wake_up_poll(x, m)                                                        \
        __wake_up(x, TASK_NORMAL, 1, poll_to_key(m))
#define wake_up_poll_on_current_cpu(x, m)                                        \
        __wake_up_on_current_cpu(x, TASK_NORMAL, poll_to_key(m))
#define wake_up_locked_poll(x, m)                                                \
        __wake_up_locked_key((x), TASK_NORMAL, poll_to_key(m))
#define wake_up_interruptible_poll(x, m)                                        \
        __wake_up(x, TASK_INTERRUPTIBLE, 1, poll_to_key(m))
#define wake_up_interruptible_sync_poll(x, m)                                        \
        __wake_up_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))
#define wake_up_interruptible_sync_poll_locked(x, m)                                \
        __wake_up_locked_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))

/**
 * wake_up_pollfree - signal that a polled waitqueue is going away
 * @wq_head: the wait queue head
 *
 * In the very rare cases where a ->poll() implementation uses a waitqueue whose
 * lifetime is tied to a task rather than to the 'struct file' being polled,
 * this function must be called before the waitqueue is freed so that
 * non-blocking polls (e.g. epoll) are notified that the queue is going away.
 *
 * The caller must also RCU-delay the freeing of the wait_queue_head, e.g. via
 * an explicit synchronize_rcu() or call_rcu(), or via SLAB_TYPESAFE_BY_RCU.
 */
static inline void wake_up_pollfree(struct wait_queue_head *wq_head)
{
        /*
         * For performance reasons, we don't always take the queue lock here.
         * Therefore, we might race with someone removing the last entry from
         * the queue, and proceed while they still hold the queue lock.
         * However, rcu_read_lock() is required to be held in such cases, so we
         * can safely proceed with an RCU-delayed free.
         */
        if (waitqueue_active(wq_head))
                __wake_up_pollfree(wq_head);
}

#define ___wait_cond_timeout(condition)                                                \
({                                                                                \
        bool __cond = (condition);                                                \
        if (__cond && !__ret)                                                        \
                __ret = 1;                                                        \
        __cond || !__ret;                                                        \
})

#define ___wait_is_interruptible(state)                                                \
        (!__builtin_constant_p(state) ||                                        \
         (state & (TASK_INTERRUPTIBLE | TASK_WAKEKILL)))

extern void init_wait_entry(struct wait_queue_entry *wq_entry, int flags);

/*
 * The below macro ___wait_event() has an explicit shadow of the __ret
 * variable when used from the wait_event_*() macros.
 *
 * This is so that both can use the ___wait_cond_timeout() construct
 * to wrap the condition.
 *
 * The type inconsistency of the wait_event_*() __ret variable is also
 * on purpose; we use long where we can return timeout values and int
 * otherwise.
 */

#define ___wait_event(wq_head, condition, state, exclusive, ret, cmd)                \
({                                                                                \
        __label__ __out;                                                        \
        struct wait_queue_entry __wq_entry;                                        \
        long __ret = ret;        /* explicit shadow */                                \
                                                                                \
        init_wait_entry(&__wq_entry, exclusive ? WQ_FLAG_EXCLUSIVE : 0);        \
        for (;;) {                                                                \
                long __int = prepare_to_wait_event(&wq_head, &__wq_entry, state);\
                                                                                \
                if (condition)                                                        \
                        break;                                                        \
                                                                                \
                if (___wait_is_interruptible(state) && __int) {                        \
                        __ret = __int;                                                \
                        goto __out;                                                \
                }                                                                \
                                                                                \
                cmd;                                                                \
                                                                                \
                if (condition)                                                        \
                        break;                                                        \
        }                                                                        \
        finish_wait(&wq_head, &__wq_entry);                                        \
__out:        __ret;                                                                        \
})

#define __wait_event(wq_head, condition)                                        \
        (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,        \
                            schedule())

/**
 * wait_event - sleep until a condition gets true
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 */
#define wait_event(wq_head, condition)                                                \
do {                                                                                \
        might_sleep();                                                                \
        if (condition)                                                                \
                break;                                                                \
        __wait_event(wq_head, condition);                                        \
} while (0)

#define __io_wait_event(wq_head, condition)                                        \
        (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,        \
                            io_schedule())

/*
 * io_wait_event() -- like wait_event() but with io_schedule()
 */
#define io_wait_event(wq_head, condition)                                        \
do {                                                                                \
        might_sleep();                                                                \
        if (condition)                                                                \
                break;                                                                \
        __io_wait_event(wq_head, condition);                                        \
} while (0)

#define __wait_event_freezable(wq_head, condition)                                \
        ___wait_event(wq_head, condition, (TASK_INTERRUPTIBLE|TASK_FREEZABLE),        \
                        0, 0, schedule())

/**
 * wait_event_freezable - sleep (or freeze) until a condition gets true
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE -- so as not to contribute
 * to system load) until the @condition evaluates to true. The
 * @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 */
#define wait_event_freezable(wq_head, condition)                                \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_freezable(wq_head, condition);                \
        __ret;                                                                        \
})

#define __wait_event_timeout(wq_head, condition, timeout)                        \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      TASK_UNINTERRUPTIBLE, 0, timeout,                                \
                      __ret = schedule_timeout(__ret))

/**
 * wait_event_timeout - sleep until a condition gets true or a timeout elapses
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, in jiffies
 *
 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * Returns:
 * 0 if the @condition evaluated to %false after the @timeout elapsed,
 * 1 if the @condition evaluated to %true after the @timeout elapsed,
 * or the remaining jiffies (at least 1) if the @condition evaluated
 * to %true before the @timeout elapsed.
 */
#define wait_event_timeout(wq_head, condition, timeout)                                \
({                                                                                \
        long __ret = timeout;                                                        \
        might_sleep();                                                                \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_timeout(wq_head, condition, timeout);        \
        __ret;                                                                        \
})

#define __wait_event_freezable_timeout(wq_head, condition, timeout)                \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      (TASK_INTERRUPTIBLE|TASK_FREEZABLE), 0, timeout,                \
                      __ret = schedule_timeout(__ret))

/*
 * like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid
 * increasing load and is freezable.
 */
#define wait_event_freezable_timeout(wq_head, condition, timeout)                \
({                                                                                \
        long __ret = timeout;                                                        \
        might_sleep();                                                                \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_freezable_timeout(wq_head, condition, timeout); \
        __ret;                                                                        \
})

#define __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2)                \
        (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 1, 0,        \
                            cmd1; schedule(); cmd2)
/*
 * Just like wait_event_cmd(), except it sets exclusive flag
 */
#define wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2)                \
do {                                                                                \
        if (condition)                                                                \
                break;                                                                \
        __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2);                \
} while (0)

#define __wait_event_cmd(wq_head, condition, cmd1, cmd2)                        \
        (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,        \
                            cmd1; schedule(); cmd2)

/**
 * wait_event_cmd - sleep until a condition gets true
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @cmd1: the command will be executed before sleep
 * @cmd2: the command will be executed after sleep
 *
 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 */
#define wait_event_cmd(wq_head, condition, cmd1, cmd2)                                \
do {                                                                                \
        if (condition)                                                                \
                break;                                                                \
        __wait_event_cmd(wq_head, condition, cmd1, cmd2);                        \
} while (0)

#define __wait_event_interruptible(wq_head, condition)                                \
        ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0,                \
                      schedule())

/**
 * wait_event_interruptible - sleep until a condition gets true
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a
 * signal and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible(wq_head, condition)                                \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_interruptible(wq_head, condition);                \
        __ret;                                                                        \
})

#define __wait_event_interruptible_timeout(wq_head, condition, timeout)                \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      TASK_INTERRUPTIBLE, 0, timeout,                                \
                      __ret = schedule_timeout(__ret))

/**
 * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, in jiffies
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * Returns:
 * 0 if the @condition evaluated to %false after the @timeout elapsed,
 * 1 if the @condition evaluated to %true after the @timeout elapsed,
 * the remaining jiffies (at least 1) if the @condition evaluated
 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
 * interrupted by a signal.
 */
#define wait_event_interruptible_timeout(wq_head, condition, timeout)                \
({                                                                                \
        long __ret = timeout;                                                        \
        might_sleep();                                                                \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_interruptible_timeout(wq_head,                \
                                                condition, timeout);                \
        __ret;                                                                        \
})

#define __wait_event_hrtimeout(wq_head, condition, timeout, state)                \
({                                                                                \
        int __ret = 0;                                                                \
        struct hrtimer_sleeper __t;                                                \
                                                                                \
        hrtimer_setup_sleeper_on_stack(&__t, CLOCK_MONOTONIC,                        \
                                       HRTIMER_MODE_REL);                        \
        if ((timeout) != KTIME_MAX) {                                                \
                hrtimer_set_expires_range_ns(&__t.timer, timeout,                \
                                        current->timer_slack_ns);                \
                hrtimer_sleeper_start_expires(&__t, HRTIMER_MODE_REL);                \
        }                                                                        \
                                                                                \
        __ret = ___wait_event(wq_head, condition, state, 0, 0,                        \
                if (!__t.task) {                                                \
                        __ret = -ETIME;                                                \
                        break;                                                        \
                }                                                                \
                schedule());                                                        \
                                                                                \
        hrtimer_cancel(&__t.timer);                                                \
        destroy_hrtimer_on_stack(&__t.timer);                                        \
        __ret;                                                                        \
})

/**
 * wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, as a ktime_t
 *
 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function returns 0 if @condition became true, or -ETIME if the timeout
 * elapsed.
 */
#define wait_event_hrtimeout(wq_head, condition, timeout)                        \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_hrtimeout(wq_head, condition, timeout,        \
                                               TASK_UNINTERRUPTIBLE);                \
        __ret;                                                                        \
})

/**
 * wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
 * @wq: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, as a ktime_t
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function returns 0 if @condition became true, -ERESTARTSYS if it was
 * interrupted by a signal, or -ETIME if the timeout elapsed.
 */
#define wait_event_interruptible_hrtimeout(wq, condition, timeout)                \
({                                                                                \
        long __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_hrtimeout(wq, condition, timeout,                \
                                               TASK_INTERRUPTIBLE);                \
        __ret;                                                                        \
})

#define __wait_event_interruptible_exclusive(wq, condition)                        \
        ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0,                        \
                      schedule())

#define wait_event_interruptible_exclusive(wq, condition)                        \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_interruptible_exclusive(wq, condition);        \
        __ret;                                                                        \
})

#define __wait_event_killable_exclusive(wq, condition)                                \
        ___wait_event(wq, condition, TASK_KILLABLE, 1, 0,                        \
                      schedule())

#define wait_event_killable_exclusive(wq, condition)                                \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_killable_exclusive(wq, condition);                \
        __ret;                                                                        \
})


#define __wait_event_freezable_exclusive(wq, condition)                                \
        ___wait_event(wq, condition, (TASK_INTERRUPTIBLE|TASK_FREEZABLE), 1, 0,\
                        schedule())

#define wait_event_freezable_exclusive(wq, condition)                                \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_freezable_exclusive(wq, condition);        \
        __ret;                                                                        \
})

/**
 * wait_event_idle - wait for a condition without contributing to system load
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_IDLE) until the
 * @condition evaluates to true.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 */
#define wait_event_idle(wq_head, condition)                                        \
do {                                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                ___wait_event(wq_head, condition, TASK_IDLE, 0, 0, schedule());        \
} while (0)

/**
 * wait_event_idle_exclusive - wait for a condition with contributing to system load
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_IDLE) until the
 * @condition evaluates to true.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 * set thus if other processes wait on the same list, when this
 * process is woken further processes are not considered.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 */
#define wait_event_idle_exclusive(wq_head, condition)                                \
do {                                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                ___wait_event(wq_head, condition, TASK_IDLE, 1, 0, schedule());        \
} while (0)

#define __wait_event_idle_timeout(wq_head, condition, timeout)                        \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      TASK_IDLE, 0, timeout,                                        \
                      __ret = schedule_timeout(__ret))

/**
 * wait_event_idle_timeout - sleep without load until a condition becomes true or a timeout elapses
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, in jiffies
 *
 * The process is put to sleep (TASK_IDLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * Returns:
 * 0 if the @condition evaluated to %false after the @timeout elapsed,
 * 1 if the @condition evaluated to %true after the @timeout elapsed,
 * or the remaining jiffies (at least 1) if the @condition evaluated
 * to %true before the @timeout elapsed.
 */
#define wait_event_idle_timeout(wq_head, condition, timeout)                        \
({                                                                                \
        long __ret = timeout;                                                        \
        might_sleep();                                                                \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_idle_timeout(wq_head, condition, timeout);        \
        __ret;                                                                        \
})

#define __wait_event_idle_exclusive_timeout(wq_head, condition, timeout)        \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      TASK_IDLE, 1, timeout,                                        \
                      __ret = schedule_timeout(__ret))

/**
 * wait_event_idle_exclusive_timeout - sleep without load until a condition becomes true or a timeout elapses
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, in jiffies
 *
 * The process is put to sleep (TASK_IDLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 * set thus if other processes wait on the same list, when this
 * process is woken further processes are not considered.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * Returns:
 * 0 if the @condition evaluated to %false after the @timeout elapsed,
 * 1 if the @condition evaluated to %true after the @timeout elapsed,
 * or the remaining jiffies (at least 1) if the @condition evaluated
 * to %true before the @timeout elapsed.
 */
#define wait_event_idle_exclusive_timeout(wq_head, condition, timeout)                \
({                                                                                \
        long __ret = timeout;                                                        \
        might_sleep();                                                                \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_idle_exclusive_timeout(wq_head, condition, timeout);\
        __ret;                                                                        \
})

extern int do_wait_intr(wait_queue_head_t *, wait_queue_entry_t *);
extern int do_wait_intr_irq(wait_queue_head_t *, wait_queue_entry_t *);

#define __wait_event_interruptible_locked(wq, condition, exclusive, fn)                \
({                                                                                \
        int __ret;                                                                \
        DEFINE_WAIT(__wait);                                                        \
        if (exclusive)                                                                \
                __wait.flags |= WQ_FLAG_EXCLUSIVE;                                \
        do {                                                                        \
                __ret = fn(&(wq), &__wait);                                        \
                if (__ret)                                                        \
                        break;                                                        \
        } while (!(condition));                                                        \
        __remove_wait_queue(&(wq), &__wait);                                        \
        __set_current_state(TASK_RUNNING);                                        \
        __ret;                                                                        \
})


/**
 * wait_event_interruptible_locked - sleep until a condition gets true
 * @wq: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq is woken up.
 *
 * It must be called with wq.lock being held.  This spinlock is
 * unlocked while sleeping but @condition testing is done while lock
 * is held and when this macro exits the lock is held.
 *
 * The lock is locked/unlocked using spin_lock()/spin_unlock()
 * functions which must match the way they are locked/unlocked outside
 * of this macro.
 *
 * wake_up_locked() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a
 * signal and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible_locked(wq, condition)                                \
        ((condition)                                                                \
         ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr))

/**
 * wait_event_interruptible_locked_irq - sleep until a condition gets true
 * @wq: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq is woken up.
 *
 * It must be called with wq.lock being held.  This spinlock is
 * unlocked while sleeping but @condition testing is done while lock
 * is held and when this macro exits the lock is held.
 *
 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
 * functions which must match the way they are locked/unlocked outside
 * of this macro.
 *
 * wake_up_locked() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a
 * signal and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible_locked_irq(wq, condition)                        \
        ((condition)                                                                \
         ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr_irq))

/**
 * wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
 * @wq: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq is woken up.
 *
 * It must be called with wq.lock being held.  This spinlock is
 * unlocked while sleeping but @condition testing is done while lock
 * is held and when this macro exits the lock is held.
 *
 * The lock is locked/unlocked using spin_lock()/spin_unlock()
 * functions which must match the way they are locked/unlocked outside
 * of this macro.
 *
 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 * set thus when other process waits process on the list if this
 * process is awaken further processes are not considered.
 *
 * wake_up_locked() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a
 * signal and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible_exclusive_locked(wq, condition)                \
        ((condition)                                                                \
         ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr))

/**
 * wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
 * @wq: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq is woken up.
 *
 * It must be called with wq.lock being held.  This spinlock is
 * unlocked while sleeping but @condition testing is done while lock
 * is held and when this macro exits the lock is held.
 *
 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
 * functions which must match the way they are locked/unlocked outside
 * of this macro.
 *
 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 * set thus when other process waits process on the list if this
 * process is awaken further processes are not considered.
 *
 * wake_up_locked() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a
 * signal and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible_exclusive_locked_irq(wq, condition)                \
        ((condition)                                                                \
         ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr_irq))


#define __wait_event_killable(wq, condition)                                        \
        ___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())

/**
 * wait_event_killable - sleep until a condition gets true
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_KILLABLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a
 * signal and 0 if @condition evaluated to true.
 */
#define wait_event_killable(wq_head, condition)                                        \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_killable(wq_head, condition);                \
        __ret;                                                                        \
})

#define __wait_event_state(wq, condition, state)                                \
        ___wait_event(wq, condition, state, 0, 0, schedule())

/**
 * wait_event_state - sleep until a condition gets true
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @state: state to sleep in
 *
 * The process is put to sleep (@state) until the @condition evaluates to true
 * or a signal is received (when allowed by @state).  The @condition is checked
 * each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a signal
 * (when allowed by @state) and 0 if @condition evaluated to true.
 */
#define wait_event_state(wq_head, condition, state)                                \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_state(wq_head, condition, state);                \
        __ret;                                                                        \
})

#define __wait_event_killable_timeout(wq_head, condition, timeout)                \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      TASK_KILLABLE, 0, timeout,                                \
                      __ret = schedule_timeout(__ret))

/**
 * wait_event_killable_timeout - sleep until a condition gets true or a timeout elapses
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, in jiffies
 *
 * The process is put to sleep (TASK_KILLABLE) until the
 * @condition evaluates to true or a kill signal is received.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * Returns:
 * 0 if the @condition evaluated to %false after the @timeout elapsed,
 * 1 if the @condition evaluated to %true after the @timeout elapsed,
 * the remaining jiffies (at least 1) if the @condition evaluated
 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
 * interrupted by a kill signal.
 *
 * Only kill signals interrupt this process.
 */
#define wait_event_killable_timeout(wq_head, condition, timeout)                \
({                                                                                \
        long __ret = timeout;                                                        \
        might_sleep();                                                                \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_killable_timeout(wq_head,                        \
                                                condition, timeout);                \
        __ret;                                                                        \
})


#define __wait_event_lock_irq(wq_head, condition, lock, cmd)                        \
        (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,        \
                            spin_unlock_irq(&lock);                                \
                            cmd;                                                \
                            schedule();                                                \
                            spin_lock_irq(&lock))

/**
 * wait_event_lock_irq_cmd - sleep until a condition gets true. The
 *                             condition is checked under the lock. This
 *                             is expected to be called with the lock
 *                             taken.
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @lock: a locked spinlock_t, which will be released before cmd
 *          and schedule() and reacquired afterwards.
 * @cmd: a command which is invoked outside the critical section before
 *         sleep
 *
 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * This is supposed to be called while holding the lock. The lock is
 * dropped before invoking the cmd and going to sleep and is reacquired
 * afterwards.
 */
#define wait_event_lock_irq_cmd(wq_head, condition, lock, cmd)                        \
do {                                                                                \
        if (condition)                                                                \
                break;                                                                \
        __wait_event_lock_irq(wq_head, condition, lock, cmd);                        \
} while (0)

/**
 * wait_event_lock_irq - sleep until a condition gets true. The
 *                         condition is checked under the lock. This
 *                         is expected to be called with the lock
 *                         taken.
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @lock: a locked spinlock_t, which will be released before schedule()
 *          and reacquired afterwards.
 *
 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * This is supposed to be called while holding the lock. The lock is
 * dropped before going to sleep and is reacquired afterwards.
 */
#define wait_event_lock_irq(wq_head, condition, lock)                                \
do {                                                                                \
        if (condition)                                                                \
                break;                                                                \
        __wait_event_lock_irq(wq_head, condition, lock, );                        \
} while (0)


#define __wait_event_interruptible_lock_irq(wq_head, condition, lock, cmd)        \
        ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0,                \
                      spin_unlock_irq(&lock);                                        \
                      cmd;                                                        \
                      schedule();                                                \
                      spin_lock_irq(&lock))

/**
 * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
 *                The condition is checked under the lock. This is expected to
 *                be called with the lock taken.
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @lock: a locked spinlock_t, which will be released before cmd and
 *          schedule() and reacquired afterwards.
 * @cmd: a command which is invoked outside the critical section before
 *         sleep
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received. The @condition is
 * checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * This is supposed to be called while holding the lock. The lock is
 * dropped before invoking the cmd and going to sleep and is reacquired
 * afterwards.
 *
 * The macro will return -ERESTARTSYS if it was interrupted by a signal
 * and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible_lock_irq_cmd(wq_head, condition, lock, cmd)        \
({                                                                                \
        int __ret = 0;                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_interruptible_lock_irq(wq_head,                \
                                                condition, lock, cmd);                \
        __ret;                                                                        \
})

/**
 * wait_event_interruptible_lock_irq - sleep until a condition gets true.
 *                The condition is checked under the lock. This is expected
 *                to be called with the lock taken.
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @lock: a locked spinlock_t, which will be released before schedule()
 *          and reacquired afterwards.
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or signal is received. The @condition is
 * checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * This is supposed to be called while holding the lock. The lock is
 * dropped before going to sleep and is reacquired afterwards.
 *
 * The macro will return -ERESTARTSYS if it was interrupted by a signal
 * and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible_lock_irq(wq_head, condition, lock)                \
({                                                                                \
        int __ret = 0;                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_interruptible_lock_irq(wq_head,                \
                                                condition, lock,);                \
        __ret;                                                                        \
})

#define __wait_event_lock_irq_timeout(wq_head, condition, lock, timeout, state)        \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      state, 0, timeout,                                        \
                      spin_unlock_irq(&lock);                                        \
                      __ret = schedule_timeout(__ret);                                \
                      spin_lock_irq(&lock));

/**
 * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
 *                true or a timeout elapses. The condition is checked under
 *                the lock. This is expected to be called with the lock taken.
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @lock: a locked spinlock_t, which will be released before schedule()
 *          and reacquired afterwards.
 * @timeout: timeout, in jiffies
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or signal is received. The @condition is
 * checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * This is supposed to be called while holding the lock. The lock is
 * dropped before going to sleep and is reacquired afterwards.
 *
 * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
 * was interrupted by a signal, and the remaining jiffies otherwise
 * if the condition evaluated to true before the timeout elapsed.
 */
#define wait_event_interruptible_lock_irq_timeout(wq_head, condition, lock,        \
                                                  timeout)                        \
({                                                                                \
        long __ret = timeout;                                                        \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_lock_irq_timeout(                                \
                                        wq_head, condition, lock, timeout,        \
                                        TASK_INTERRUPTIBLE);                        \
        __ret;                                                                        \
})

#define wait_event_lock_irq_timeout(wq_head, condition, lock, timeout)                \
({                                                                                \
        long __ret = timeout;                                                        \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_lock_irq_timeout(                                \
                                        wq_head, condition, lock, timeout,        \
                                        TASK_UNINTERRUPTIBLE);                        \
        __ret;                                                                        \
})

/*
 * Waitqueues which are removed from the waitqueue_head at wakeup time
 */
void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
bool prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout);
int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);

#define DEFINE_WAIT_FUNC(name, function)                                        \
        struct wait_queue_entry name = {                                        \
                .private        = current,                                        \
                .func                = function,                                        \
                .entry                = LIST_HEAD_INIT((name).entry),                        \
        }

#define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)

#define init_wait_func(wait, function)                                                \
        do {                                                                        \
                (wait)->private = current;                                        \
                (wait)->func = function;                                        \
                INIT_LIST_HEAD(&(wait)->entry);                                        \
                (wait)->flags = 0;                                                \
        } while (0)

#define init_wait(wait)        init_wait_func(wait, autoremove_wake_function)

typedef int (*task_call_f)(struct task_struct *p, void *arg);
extern int task_call_func(struct task_struct *p, task_call_f func, void *arg);

#endif /* _LINUX_WAIT_H */


























    3 






















    5 






















    5 



    5 


    1 



    4 

    3 
















    5 



    5 



































    7 



    1 


    1 
    5 






























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
// SPDX-License-Identifier: GPL-2.0-only
/*
 * vsock sock_diag(7) module
 *
 * Copyright (C) 2017 Red Hat, Inc.
 * Author: Stefan Hajnoczi <stefanha@redhat.com>
 */

#include <linux/module.h>
#include <linux/sock_diag.h>
#include <linux/vm_sockets_diag.h>
#include <net/af_vsock.h>

static int sk_diag_fill(struct sock *sk, struct sk_buff *skb,
                        u32 portid, u32 seq, u32 flags)
{
        struct vsock_sock *vsk = vsock_sk(sk);
        struct vsock_diag_msg *rep;
        struct nlmsghdr *nlh;

        nlh = nlmsg_put(skb, portid, seq, SOCK_DIAG_BY_FAMILY, sizeof(*rep),
                        flags);
        if (!nlh)
                return -EMSGSIZE;

        rep = nlmsg_data(nlh);
        rep->vdiag_family = AF_VSOCK;

        /* Lock order dictates that sk_lock is acquired before
         * vsock_table_lock, so we cannot lock here.  Simply don't take
         * sk_lock; sk is guaranteed to stay alive since vsock_table_lock is
         * held.
         */
        rep->vdiag_type = sk->sk_type;
        rep->vdiag_state = sk->sk_state;
        rep->vdiag_shutdown = sk->sk_shutdown;
        rep->vdiag_src_cid = vsk->local_addr.svm_cid;
        rep->vdiag_src_port = vsk->local_addr.svm_port;
        rep->vdiag_dst_cid = vsk->remote_addr.svm_cid;
        rep->vdiag_dst_port = vsk->remote_addr.svm_port;
        rep->vdiag_ino = sock_i_ino(sk);

        sock_diag_save_cookie(sk, rep->vdiag_cookie);

        return 0;
}

static int vsock_diag_dump(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct vsock_diag_req *req;
        struct vsock_sock *vsk;
        unsigned int bucket;
        unsigned int last_i;
        unsigned int table;
        struct net *net;
        unsigned int i;

        req = nlmsg_data(cb->nlh);
        net = sock_net(skb->sk);

        /* State saved between calls: */
        table = cb->args[0];
        bucket = cb->args[1];
        i = last_i = cb->args[2];

        /* TODO VMCI pending sockets? */

        spin_lock_bh(&vsock_table_lock);

        /* Bind table (locally created sockets) */
        if (table == 0) {
                while (bucket < ARRAY_SIZE(vsock_bind_table)) {
                        struct list_head *head = &vsock_bind_table[bucket];

                        i = 0;
                        list_for_each_entry(vsk, head, bound_table) {
                                struct sock *sk = sk_vsock(vsk);

                                if (!net_eq(sock_net(sk), net))
                                        continue;
                                if (i < last_i)
                                        goto next_bind;
                                if (!(req->vdiag_states & (1 << sk->sk_state)))
                                        goto next_bind;
                                if (sk_diag_fill(sk, skb,
                                                 NETLINK_CB(cb->skb).portid,
                                                 cb->nlh->nlmsg_seq,
                                                 NLM_F_MULTI) < 0)
                                        goto done;
next_bind:
                                i++;
                        }
                        last_i = 0;
                        bucket++;
                }

                table++;
                bucket = 0;
        }

        /* Connected table (accepted connections) */
        while (bucket < ARRAY_SIZE(vsock_connected_table)) {
                struct list_head *head = &vsock_connected_table[bucket];

                i = 0;
                list_for_each_entry(vsk, head, connected_table) {
                        struct sock *sk = sk_vsock(vsk);

                        /* Skip sockets we've already seen above */
                        if (__vsock_in_bound_table(vsk))
                                continue;

                        if (!net_eq(sock_net(sk), net))
                                continue;
                        if (i < last_i)
                                goto next_connected;
                        if (!(req->vdiag_states & (1 << sk->sk_state)))
                                goto next_connected;
                        if (sk_diag_fill(sk, skb,
                                         NETLINK_CB(cb->skb).portid,
                                         cb->nlh->nlmsg_seq,
                                         NLM_F_MULTI) < 0)
                                goto done;
next_connected:
                        i++;
                }
                last_i = 0;
                bucket++;
        }

done:
        spin_unlock_bh(&vsock_table_lock);

        cb->args[0] = table;
        cb->args[1] = bucket;
        cb->args[2] = i;

        return skb->len;
}

static int vsock_diag_handler_dump(struct sk_buff *skb, struct nlmsghdr *h)
{
        int hdrlen = sizeof(struct vsock_diag_req);
        struct net *net = sock_net(skb->sk);

        if (nlmsg_len(h) < hdrlen)
                return -EINVAL;

        if (h->nlmsg_flags & NLM_F_DUMP) {
                struct netlink_dump_control c = {
                        .dump = vsock_diag_dump,
                };
                return netlink_dump_start(net->diag_nlsk, skb, h, &c);
        }

        return -EOPNOTSUPP;
}

static const struct sock_diag_handler vsock_diag_handler = {
        .owner = THIS_MODULE,
        .family = AF_VSOCK,
        .dump = vsock_diag_handler_dump,
};

static int __init vsock_diag_init(void)
{
        return sock_diag_register(&vsock_diag_handler);
}

static void __exit vsock_diag_exit(void)
{
        sock_diag_unregister(&vsock_diag_handler);
}

module_init(vsock_diag_init);
module_exit(vsock_diag_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("VMware Virtual Sockets monitoring via SOCK_DIAG");
MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG,
                               40 /* AF_VSOCK */);











































































   12 





    3 


































































    2 










































    2 


    2 
    1 


    1 
    2 



    2 













    2 
    2 








    2 
















    2 
    2 
    2 



































    2 






    2 





    2 








   12 








    2 















   12 




   12 




   12 












    6 
    6 
    6 

    6 
    6 












   12 



   12 





   12 














    2 













   12 

   12 







   12 











   12 



   12 
   12 
   12 


    1 
   12 




























    1 
    1 
    1 





    1 























    6 






    6 

    6 













    6 


    6 
    1 

    6 




   12 











   12 































































    3 









    3 
    3 







    3 

    3 


    3 




















































































































































































































































 1023 





 1009 





















   29 



























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
// SPDX-License-Identifier: GPL-2.0-only
// Copyright (c) 2020 Facebook Inc.

#include <linux/ethtool_netlink.h>
#include <linux/netdevice.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/workqueue.h>
#include <net/udp_tunnel.h>
#include <net/vxlan.h>

enum udp_tunnel_nic_table_entry_flags {
        UDP_TUNNEL_NIC_ENTRY_ADD        = BIT(0),
        UDP_TUNNEL_NIC_ENTRY_DEL        = BIT(1),
        UDP_TUNNEL_NIC_ENTRY_OP_FAIL        = BIT(2),
        UDP_TUNNEL_NIC_ENTRY_FROZEN        = BIT(3),
};

struct udp_tunnel_nic_table_entry {
        __be16 port;
        u8 type;
        u8 flags;
        u16 use_cnt;
#define UDP_TUNNEL_NIC_USE_CNT_MAX        U16_MAX
        u8 hw_priv;
};

/**
 * struct udp_tunnel_nic - UDP tunnel port offload state
 * @work:        async work for talking to hardware from process context
 * @dev:        netdev pointer
 * @need_sync:        at least one port start changed
 * @need_replay: space was freed, we need a replay of all ports
 * @work_pending: @work is currently scheduled
 * @n_tables:        number of tables under @entries
 * @missed:        bitmap of tables which overflown
 * @entries:        table of tables of ports currently offloaded
 */
struct udp_tunnel_nic {
        struct work_struct work;

        struct net_device *dev;

        u8 need_sync:1;
        u8 need_replay:1;
        u8 work_pending:1;

        unsigned int n_tables;
        unsigned long missed;
        struct udp_tunnel_nic_table_entry *entries[] __counted_by(n_tables);
};

/* We ensure all work structs are done using driver state, but not the code.
 * We need a workqueue we can flush before module gets removed.
 */
static struct workqueue_struct *udp_tunnel_nic_workqueue;

static const char *udp_tunnel_nic_tunnel_type_name(unsigned int type)
{
        switch (type) {
        case UDP_TUNNEL_TYPE_VXLAN:
                return "vxlan";
        case UDP_TUNNEL_TYPE_GENEVE:
                return "geneve";
        case UDP_TUNNEL_TYPE_VXLAN_GPE:
                return "vxlan-gpe";
        default:
                return "unknown";
        }
}

static bool
udp_tunnel_nic_entry_is_free(struct udp_tunnel_nic_table_entry *entry)
{
        return entry->use_cnt == 0 && !entry->flags;
}

static bool
udp_tunnel_nic_entry_is_present(struct udp_tunnel_nic_table_entry *entry)
{
        return entry->use_cnt && !(entry->flags & ~UDP_TUNNEL_NIC_ENTRY_FROZEN);
}

static bool
udp_tunnel_nic_entry_is_frozen(struct udp_tunnel_nic_table_entry *entry)
{
        return entry->flags & UDP_TUNNEL_NIC_ENTRY_FROZEN;
}

static void
udp_tunnel_nic_entry_freeze_used(struct udp_tunnel_nic_table_entry *entry)
{
        if (!udp_tunnel_nic_entry_is_free(entry))
                entry->flags |= UDP_TUNNEL_NIC_ENTRY_FROZEN;
}

static void
udp_tunnel_nic_entry_unfreeze(struct udp_tunnel_nic_table_entry *entry)
{
        entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_FROZEN;
}

static bool
udp_tunnel_nic_entry_is_queued(struct udp_tunnel_nic_table_entry *entry)
{
        return entry->flags & (UDP_TUNNEL_NIC_ENTRY_ADD |
                               UDP_TUNNEL_NIC_ENTRY_DEL);
}

static void
udp_tunnel_nic_entry_queue(struct udp_tunnel_nic *utn,
                           struct udp_tunnel_nic_table_entry *entry,
                           unsigned int flag)
{
        entry->flags |= flag;
        utn->need_sync = 1;
}

static void
udp_tunnel_nic_ti_from_entry(struct udp_tunnel_nic_table_entry *entry,
                             struct udp_tunnel_info *ti)
{
        memset(ti, 0, sizeof(*ti));
        ti->port = entry->port;
        ti->type = entry->type;
        ti->hw_priv = entry->hw_priv;
}

static bool
udp_tunnel_nic_is_empty(struct net_device *dev, struct udp_tunnel_nic *utn)
{
        const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
        unsigned int i, j;

        for (i = 0; i < utn->n_tables; i++)
                for (j = 0; j < info->tables[i].n_entries; j++)
                        if (!udp_tunnel_nic_entry_is_free(&utn->entries[i][j]))
                                return false;
        return true;
}

static bool
udp_tunnel_nic_should_replay(struct net_device *dev, struct udp_tunnel_nic *utn)
{
        const struct udp_tunnel_nic_table_info *table;
        unsigned int i, j;

        if (!utn->missed)
                return false;

        for (i = 0; i < utn->n_tables; i++) {
                table = &dev->udp_tunnel_nic_info->tables[i];
                if (!test_bit(i, &utn->missed))
                        continue;

                for (j = 0; j < table->n_entries; j++)
                        if (udp_tunnel_nic_entry_is_free(&utn->entries[i][j]))
                                return true;
        }

        return false;
}

static void
__udp_tunnel_nic_get_port(struct net_device *dev, unsigned int table,
                          unsigned int idx, struct udp_tunnel_info *ti)
{
        struct udp_tunnel_nic_table_entry *entry;
        struct udp_tunnel_nic *utn;

        utn = dev->udp_tunnel_nic;
        entry = &utn->entries[table][idx];

        if (entry->use_cnt)
                udp_tunnel_nic_ti_from_entry(entry, ti);
}

static void
__udp_tunnel_nic_set_port_priv(struct net_device *dev, unsigned int table,
                               unsigned int idx, u8 priv)
{
        dev->udp_tunnel_nic->entries[table][idx].hw_priv = priv;
}

static void
udp_tunnel_nic_entry_update_done(struct udp_tunnel_nic_table_entry *entry,
                                 int err)
{
        bool dodgy = entry->flags & UDP_TUNNEL_NIC_ENTRY_OP_FAIL;

        WARN_ON_ONCE(entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD &&
                     entry->flags & UDP_TUNNEL_NIC_ENTRY_DEL);

        if (entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD &&
            (!err || (err == -EEXIST && dodgy)))
                entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_ADD;

        if (entry->flags & UDP_TUNNEL_NIC_ENTRY_DEL &&
            (!err || (err == -ENOENT && dodgy)))
                entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_DEL;

        if (!err)
                entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
        else
                entry->flags |= UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
}

static void
udp_tunnel_nic_device_sync_one(struct net_device *dev,
                               struct udp_tunnel_nic *utn,
                               unsigned int table, unsigned int idx)
{
        struct udp_tunnel_nic_table_entry *entry;
        struct udp_tunnel_info ti;
        int err;

        entry = &utn->entries[table][idx];
        if (!udp_tunnel_nic_entry_is_queued(entry))
                return;

        udp_tunnel_nic_ti_from_entry(entry, &ti);
        if (entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD)
                err = dev->udp_tunnel_nic_info->set_port(dev, table, idx, &ti);
        else
                err = dev->udp_tunnel_nic_info->unset_port(dev, table, idx,
                                                           &ti);
        udp_tunnel_nic_entry_update_done(entry, err);

        if (err)
                netdev_warn(dev,
                            "UDP tunnel port sync failed port %d type %s: %d\n",
                            be16_to_cpu(entry->port),
                            udp_tunnel_nic_tunnel_type_name(entry->type),
                            err);
}

static void
udp_tunnel_nic_device_sync_by_port(struct net_device *dev,
                                   struct udp_tunnel_nic *utn)
{
        const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
        unsigned int i, j;

        for (i = 0; i < utn->n_tables; i++)
                for (j = 0; j < info->tables[i].n_entries; j++)
                        udp_tunnel_nic_device_sync_one(dev, utn, i, j);
}

static void
udp_tunnel_nic_device_sync_by_table(struct net_device *dev,
                                    struct udp_tunnel_nic *utn)
{
        const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
        unsigned int i, j;
        int err;

        for (i = 0; i < utn->n_tables; i++) {
                /* Find something that needs sync in this table */
                for (j = 0; j < info->tables[i].n_entries; j++)
                        if (udp_tunnel_nic_entry_is_queued(&utn->entries[i][j]))
                                break;
                if (j == info->tables[i].n_entries)
                        continue;

                err = info->sync_table(dev, i);
                if (err)
                        netdev_warn(dev, "UDP tunnel port sync failed for table %d: %d\n",
                                    i, err);

                for (j = 0; j < info->tables[i].n_entries; j++) {
                        struct udp_tunnel_nic_table_entry *entry;

                        entry = &utn->entries[i][j];
                        if (udp_tunnel_nic_entry_is_queued(entry))
                                udp_tunnel_nic_entry_update_done(entry, err);
                }
        }
}

static void
__udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn)
{
        if (!utn->need_sync)
                return;

        if (dev->udp_tunnel_nic_info->sync_table)
                udp_tunnel_nic_device_sync_by_table(dev, utn);
        else
                udp_tunnel_nic_device_sync_by_port(dev, utn);

        utn->need_sync = 0;
        /* Can't replay directly here, in case we come from the tunnel driver's
         * notification - trying to replay may deadlock inside tunnel driver.
         */
        utn->need_replay = udp_tunnel_nic_should_replay(dev, utn);
}

static void
udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn)
{
        const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
        bool may_sleep;

        if (!utn->need_sync)
                return;

        /* Drivers which sleep in the callback need to update from
         * the workqueue, if we come from the tunnel driver's notification.
         */
        may_sleep = info->flags & UDP_TUNNEL_NIC_INFO_MAY_SLEEP;
        if (!may_sleep)
                __udp_tunnel_nic_device_sync(dev, utn);
        if (may_sleep || utn->need_replay) {
                queue_work(udp_tunnel_nic_workqueue, &utn->work);
                utn->work_pending = 1;
        }
}

static bool
udp_tunnel_nic_table_is_capable(const struct udp_tunnel_nic_table_info *table,
                                struct udp_tunnel_info *ti)
{
        return table->tunnel_types & ti->type;
}

static bool
udp_tunnel_nic_is_capable(struct net_device *dev, struct udp_tunnel_nic *utn,
                          struct udp_tunnel_info *ti)
{
        const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
        unsigned int i;

        /* Special case IPv4-only NICs */
        if (info->flags & UDP_TUNNEL_NIC_INFO_IPV4_ONLY &&
            ti->sa_family != AF_INET)
                return false;

        for (i = 0; i < utn->n_tables; i++)
                if (udp_tunnel_nic_table_is_capable(&info->tables[i], ti))
                        return true;
        return false;
}

static int
udp_tunnel_nic_has_collision(struct net_device *dev, struct udp_tunnel_nic *utn,
                             struct udp_tunnel_info *ti)
{
        const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
        struct udp_tunnel_nic_table_entry *entry;
        unsigned int i, j;

        for (i = 0; i < utn->n_tables; i++)
                for (j = 0; j < info->tables[i].n_entries; j++) {
                        entry =        &utn->entries[i][j];

                        if (!udp_tunnel_nic_entry_is_free(entry) &&
                            entry->port == ti->port &&
                            entry->type != ti->type) {
                                __set_bit(i, &utn->missed);
                                return true;
                        }
                }
        return false;
}

static void
udp_tunnel_nic_entry_adj(struct udp_tunnel_nic *utn,
                         unsigned int table, unsigned int idx, int use_cnt_adj)
{
        struct udp_tunnel_nic_table_entry *entry =  &utn->entries[table][idx];
        bool dodgy = entry->flags & UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
        unsigned int from, to;

        WARN_ON(entry->use_cnt + (u32)use_cnt_adj > U16_MAX);

        /* If not going from used to unused or vice versa - all done.
         * For dodgy entries make sure we try to sync again (queue the entry).
         */
        entry->use_cnt += use_cnt_adj;
        if (!dodgy && !entry->use_cnt == !(entry->use_cnt - use_cnt_adj))
                return;

        /* Cancel the op before it was sent to the device, if possible,
         * otherwise we'd need to take special care to issue commands
         * in the same order the ports arrived.
         */
        if (use_cnt_adj < 0) {
                from = UDP_TUNNEL_NIC_ENTRY_ADD;
                to = UDP_TUNNEL_NIC_ENTRY_DEL;
        } else {
                from = UDP_TUNNEL_NIC_ENTRY_DEL;
                to = UDP_TUNNEL_NIC_ENTRY_ADD;
        }

        if (entry->flags & from) {
                entry->flags &= ~from;
                if (!dodgy)
                        return;
        }

        udp_tunnel_nic_entry_queue(utn, entry, to);
}

static bool
udp_tunnel_nic_entry_try_adj(struct udp_tunnel_nic *utn,
                             unsigned int table, unsigned int idx,
                             struct udp_tunnel_info *ti, int use_cnt_adj)
{
        struct udp_tunnel_nic_table_entry *entry =  &utn->entries[table][idx];

        if (udp_tunnel_nic_entry_is_free(entry) ||
            entry->port != ti->port ||
            entry->type != ti->type)
                return false;

        if (udp_tunnel_nic_entry_is_frozen(entry))
                return true;

        udp_tunnel_nic_entry_adj(utn, table, idx, use_cnt_adj);
        return true;
}

/* Try to find existing matching entry and adjust its use count, instead of
 * adding a new one. Returns true if entry was found. In case of delete the
 * entry may have gotten removed in the process, in which case it will be
 * queued for removal.
 */
static bool
udp_tunnel_nic_try_existing(struct net_device *dev, struct udp_tunnel_nic *utn,
                            struct udp_tunnel_info *ti, int use_cnt_adj)
{
        const struct udp_tunnel_nic_table_info *table;
        unsigned int i, j;

        for (i = 0; i < utn->n_tables; i++) {
                table = &dev->udp_tunnel_nic_info->tables[i];
                if (!udp_tunnel_nic_table_is_capable(table, ti))
                        continue;

                for (j = 0; j < table->n_entries; j++)
                        if (udp_tunnel_nic_entry_try_adj(utn, i, j, ti,
                                                         use_cnt_adj))
                                return true;
        }

        return false;
}

static bool
udp_tunnel_nic_add_existing(struct net_device *dev, struct udp_tunnel_nic *utn,
                            struct udp_tunnel_info *ti)
{
        return udp_tunnel_nic_try_existing(dev, utn, ti, +1);
}

static bool
udp_tunnel_nic_del_existing(struct net_device *dev, struct udp_tunnel_nic *utn,
                            struct udp_tunnel_info *ti)
{
        return udp_tunnel_nic_try_existing(dev, utn, ti, -1);
}

static bool
udp_tunnel_nic_add_new(struct net_device *dev, struct udp_tunnel_nic *utn,
                       struct udp_tunnel_info *ti)
{
        const struct udp_tunnel_nic_table_info *table;
        unsigned int i, j;

        for (i = 0; i < utn->n_tables; i++) {
                table = &dev->udp_tunnel_nic_info->tables[i];
                if (!udp_tunnel_nic_table_is_capable(table, ti))
                        continue;

                for (j = 0; j < table->n_entries; j++) {
                        struct udp_tunnel_nic_table_entry *entry;

                        entry = &utn->entries[i][j];
                        if (!udp_tunnel_nic_entry_is_free(entry))
                                continue;

                        entry->port = ti->port;
                        entry->type = ti->type;
                        entry->use_cnt = 1;
                        udp_tunnel_nic_entry_queue(utn, entry,
                                                   UDP_TUNNEL_NIC_ENTRY_ADD);
                        return true;
                }

                /* The different table may still fit this port in, but there
                 * are no devices currently which have multiple tables accepting
                 * the same tunnel type, and false positives are okay.
                 */
                __set_bit(i, &utn->missed);
        }

        return false;
}

static void
__udp_tunnel_nic_add_port(struct net_device *dev, struct udp_tunnel_info *ti)
{
        const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
        struct udp_tunnel_nic *utn;

        utn = dev->udp_tunnel_nic;
        if (!utn)
                return;
        if (!netif_running(dev) && info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY)
                return;
        if (info->flags & UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN &&
            ti->port == htons(IANA_VXLAN_UDP_PORT)) {
                if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
                        netdev_warn(dev, "device assumes port 4789 will be used by vxlan tunnels\n");
                return;
        }

        if (!udp_tunnel_nic_is_capable(dev, utn, ti))
                return;

        /* It may happen that a tunnel of one type is removed and different
         * tunnel type tries to reuse its port before the device was informed.
         * Rely on utn->missed to re-add this port later.
         */
        if (udp_tunnel_nic_has_collision(dev, utn, ti))
                return;

        if (!udp_tunnel_nic_add_existing(dev, utn, ti))
                udp_tunnel_nic_add_new(dev, utn, ti);

        udp_tunnel_nic_device_sync(dev, utn);
}

static void
__udp_tunnel_nic_del_port(struct net_device *dev, struct udp_tunnel_info *ti)
{
        struct udp_tunnel_nic *utn;

        utn = dev->udp_tunnel_nic;
        if (!utn)
                return;

        if (!udp_tunnel_nic_is_capable(dev, utn, ti))
                return;

        udp_tunnel_nic_del_existing(dev, utn, ti);

        udp_tunnel_nic_device_sync(dev, utn);
}

static void __udp_tunnel_nic_reset_ntf(struct net_device *dev)
{
        const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
        struct udp_tunnel_nic *utn;
        unsigned int i, j;

        ASSERT_RTNL();

        utn = dev->udp_tunnel_nic;
        if (!utn)
                return;

        utn->need_sync = false;
        for (i = 0; i < utn->n_tables; i++)
                for (j = 0; j < info->tables[i].n_entries; j++) {
                        struct udp_tunnel_nic_table_entry *entry;

                        entry = &utn->entries[i][j];

                        entry->flags &= ~(UDP_TUNNEL_NIC_ENTRY_DEL |
                                          UDP_TUNNEL_NIC_ENTRY_OP_FAIL);
                        /* We don't release rtnl across ops */
                        WARN_ON(entry->flags & UDP_TUNNEL_NIC_ENTRY_FROZEN);
                        if (!entry->use_cnt)
                                continue;

                        udp_tunnel_nic_entry_queue(utn, entry,
                                                   UDP_TUNNEL_NIC_ENTRY_ADD);
                }

        __udp_tunnel_nic_device_sync(dev, utn);
}

static size_t
__udp_tunnel_nic_dump_size(struct net_device *dev, unsigned int table)
{
        const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
        struct udp_tunnel_nic *utn;
        unsigned int j;
        size_t size;

        utn = dev->udp_tunnel_nic;
        if (!utn)
                return 0;

        size = 0;
        for (j = 0; j < info->tables[table].n_entries; j++) {
                if (!udp_tunnel_nic_entry_is_present(&utn->entries[table][j]))
                        continue;

                size += nla_total_size(0) +                 /* _TABLE_ENTRY */
                        nla_total_size(sizeof(__be16)) + /* _ENTRY_PORT */
                        nla_total_size(sizeof(u32));         /* _ENTRY_TYPE */
        }

        return size;
}

static int
__udp_tunnel_nic_dump_write(struct net_device *dev, unsigned int table,
                            struct sk_buff *skb)
{
        const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
        struct udp_tunnel_nic *utn;
        struct nlattr *nest;
        unsigned int j;

        utn = dev->udp_tunnel_nic;
        if (!utn)
                return 0;

        for (j = 0; j < info->tables[table].n_entries; j++) {
                if (!udp_tunnel_nic_entry_is_present(&utn->entries[table][j]))
                        continue;

                nest = nla_nest_start(skb, ETHTOOL_A_TUNNEL_UDP_TABLE_ENTRY);
                if (!nest)
                        return -EMSGSIZE;

                if (nla_put_be16(skb, ETHTOOL_A_TUNNEL_UDP_ENTRY_PORT,
                                 utn->entries[table][j].port) ||
                    nla_put_u32(skb, ETHTOOL_A_TUNNEL_UDP_ENTRY_TYPE,
                                ilog2(utn->entries[table][j].type)))
                        goto err_cancel;

                nla_nest_end(skb, nest);
        }

        return 0;

err_cancel:
        nla_nest_cancel(skb, nest);
        return -EMSGSIZE;
}

static const struct udp_tunnel_nic_ops __udp_tunnel_nic_ops = {
        .get_port        = __udp_tunnel_nic_get_port,
        .set_port_priv        = __udp_tunnel_nic_set_port_priv,
        .add_port        = __udp_tunnel_nic_add_port,
        .del_port        = __udp_tunnel_nic_del_port,
        .reset_ntf        = __udp_tunnel_nic_reset_ntf,
        .dump_size        = __udp_tunnel_nic_dump_size,
        .dump_write        = __udp_tunnel_nic_dump_write,
};

static void
udp_tunnel_nic_flush(struct net_device *dev, struct udp_tunnel_nic *utn)
{
        const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
        unsigned int i, j;

        for (i = 0; i < utn->n_tables; i++)
                for (j = 0; j < info->tables[i].n_entries; j++) {
                        int adj_cnt = -utn->entries[i][j].use_cnt;

                        if (adj_cnt)
                                udp_tunnel_nic_entry_adj(utn, i, j, adj_cnt);
                }

        __udp_tunnel_nic_device_sync(dev, utn);

        for (i = 0; i < utn->n_tables; i++)
                memset(utn->entries[i], 0, array_size(info->tables[i].n_entries,
                                                      sizeof(**utn->entries)));
        WARN_ON(utn->need_sync);
        utn->need_replay = 0;
}

static void
udp_tunnel_nic_replay(struct net_device *dev, struct udp_tunnel_nic *utn)
{
        const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
        struct udp_tunnel_nic_shared_node *node;
        unsigned int i, j;

        /* Freeze all the ports we are already tracking so that the replay
         * does not double up the refcount.
         */
        for (i = 0; i < utn->n_tables; i++)
                for (j = 0; j < info->tables[i].n_entries; j++)
                        udp_tunnel_nic_entry_freeze_used(&utn->entries[i][j]);
        utn->missed = 0;
        utn->need_replay = 0;

        if (!info->shared) {
                udp_tunnel_get_rx_info(dev);
        } else {
                list_for_each_entry(node, &info->shared->devices, list)
                        udp_tunnel_get_rx_info(node->dev);
        }

        for (i = 0; i < utn->n_tables; i++)
                for (j = 0; j < info->tables[i].n_entries; j++)
                        udp_tunnel_nic_entry_unfreeze(&utn->entries[i][j]);
}

static void udp_tunnel_nic_device_sync_work(struct work_struct *work)
{
        struct udp_tunnel_nic *utn =
                container_of(work, struct udp_tunnel_nic, work);

        rtnl_lock();
        utn->work_pending = 0;
        __udp_tunnel_nic_device_sync(utn->dev, utn);

        if (utn->need_replay)
                udp_tunnel_nic_replay(utn->dev, utn);
        rtnl_unlock();
}

static struct udp_tunnel_nic *
udp_tunnel_nic_alloc(const struct udp_tunnel_nic_info *info,
                     unsigned int n_tables)
{
        struct udp_tunnel_nic *utn;
        unsigned int i;

        utn = kzalloc(struct_size(utn, entries, n_tables), GFP_KERNEL);
        if (!utn)
                return NULL;
        utn->n_tables = n_tables;
        INIT_WORK(&utn->work, udp_tunnel_nic_device_sync_work);

        for (i = 0; i < n_tables; i++) {
                utn->entries[i] = kcalloc(info->tables[i].n_entries,
                                          sizeof(*utn->entries[i]), GFP_KERNEL);
                if (!utn->entries[i])
                        goto err_free_prev_entries;
        }

        return utn;

err_free_prev_entries:
        while (i--)
                kfree(utn->entries[i]);
        kfree(utn);
        return NULL;
}

static void udp_tunnel_nic_free(struct udp_tunnel_nic *utn)
{
        unsigned int i;

        for (i = 0; i < utn->n_tables; i++)
                kfree(utn->entries[i]);
        kfree(utn);
}

static int udp_tunnel_nic_register(struct net_device *dev)
{
        const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
        struct udp_tunnel_nic_shared_node *node = NULL;
        struct udp_tunnel_nic *utn;
        unsigned int n_tables, i;

        BUILD_BUG_ON(sizeof(utn->missed) * BITS_PER_BYTE <
                     UDP_TUNNEL_NIC_MAX_TABLES);
        /* Expect use count of at most 2 (IPv4, IPv6) per device */
        BUILD_BUG_ON(UDP_TUNNEL_NIC_USE_CNT_MAX <
                     UDP_TUNNEL_NIC_MAX_SHARING_DEVICES * 2);

        /* Check that the driver info is sane */
        if (WARN_ON(!info->set_port != !info->unset_port) ||
            WARN_ON(!info->set_port == !info->sync_table) ||
            WARN_ON(!info->tables[0].n_entries))
                return -EINVAL;

        if (WARN_ON(info->shared &&
                    info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY))
                return -EINVAL;

        n_tables = 1;
        for (i = 1; i < UDP_TUNNEL_NIC_MAX_TABLES; i++) {
                if (!info->tables[i].n_entries)
                        continue;

                n_tables++;
                if (WARN_ON(!info->tables[i - 1].n_entries))
                        return -EINVAL;
        }

        /* Create UDP tunnel state structures */
        if (info->shared) {
                node = kzalloc(sizeof(*node), GFP_KERNEL);
                if (!node)
                        return -ENOMEM;

                node->dev = dev;
        }

        if (info->shared && info->shared->udp_tunnel_nic_info) {
                utn = info->shared->udp_tunnel_nic_info;
        } else {
                utn = udp_tunnel_nic_alloc(info, n_tables);
                if (!utn) {
                        kfree(node);
                        return -ENOMEM;
                }
        }

        if (info->shared) {
                if (!info->shared->udp_tunnel_nic_info) {
                        INIT_LIST_HEAD(&info->shared->devices);
                        info->shared->udp_tunnel_nic_info = utn;
                }

                list_add_tail(&node->list, &info->shared->devices);
        }

        utn->dev = dev;
        dev_hold(dev);
        dev->udp_tunnel_nic = utn;

        if (!(info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY))
                udp_tunnel_get_rx_info(dev);

        return 0;
}

static void
udp_tunnel_nic_unregister(struct net_device *dev, struct udp_tunnel_nic *utn)
{
        const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;

        /* For a shared table remove this dev from the list of sharing devices
         * and if there are other devices just detach.
         */
        if (info->shared) {
                struct udp_tunnel_nic_shared_node *node, *first;

                list_for_each_entry(node, &info->shared->devices, list)
                        if (node->dev == dev)
                                break;
                if (list_entry_is_head(node, &info->shared->devices, list))
                        return;

                list_del(&node->list);
                kfree(node);

                first = list_first_entry_or_null(&info->shared->devices,
                                                 typeof(*first), list);
                if (first) {
                        udp_tunnel_drop_rx_info(dev);
                        utn->dev = first->dev;
                        goto release_dev;
                }

                info->shared->udp_tunnel_nic_info = NULL;
        }

        /* Flush before we check work, so we don't waste time adding entries
         * from the work which we will boot immediately.
         */
        udp_tunnel_nic_flush(dev, utn);

        /* Wait for the work to be done using the state, netdev core will
         * retry unregister until we give up our reference on this device.
         */
        if (utn->work_pending)
                return;

        udp_tunnel_nic_free(utn);
release_dev:
        dev->udp_tunnel_nic = NULL;
        dev_put(dev);
}

static int
udp_tunnel_nic_netdevice_event(struct notifier_block *unused,
                               unsigned long event, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        const struct udp_tunnel_nic_info *info;
        struct udp_tunnel_nic *utn;

        info = dev->udp_tunnel_nic_info;
        if (!info)
                return NOTIFY_DONE;

        if (event == NETDEV_REGISTER) {
                int err;

                err = udp_tunnel_nic_register(dev);
                if (err)
                        netdev_WARN(dev, "failed to register for UDP tunnel offloads: %d", err);
                return notifier_from_errno(err);
        }
        /* All other events will need the udp_tunnel_nic state */
        utn = dev->udp_tunnel_nic;
        if (!utn)
                return NOTIFY_DONE;

        if (event == NETDEV_UNREGISTER) {
                udp_tunnel_nic_unregister(dev, utn);
                return NOTIFY_OK;
        }

        /* All other events only matter if NIC has to be programmed open */
        if (!(info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY))
                return NOTIFY_DONE;

        if (event == NETDEV_UP) {
                WARN_ON(!udp_tunnel_nic_is_empty(dev, utn));
                udp_tunnel_get_rx_info(dev);
                return NOTIFY_OK;
        }
        if (event == NETDEV_GOING_DOWN) {
                udp_tunnel_nic_flush(dev, utn);
                return NOTIFY_OK;
        }

        return NOTIFY_DONE;
}

static struct notifier_block udp_tunnel_nic_notifier_block __read_mostly = {
        .notifier_call = udp_tunnel_nic_netdevice_event,
};

static int __init udp_tunnel_nic_init_module(void)
{
        int err;

        udp_tunnel_nic_workqueue = alloc_ordered_workqueue("udp_tunnel_nic", 0);
        if (!udp_tunnel_nic_workqueue)
                return -ENOMEM;

        rtnl_lock();
        udp_tunnel_nic_ops = &__udp_tunnel_nic_ops;
        rtnl_unlock();

        err = register_netdevice_notifier(&udp_tunnel_nic_notifier_block);
        if (err)
                goto err_unset_ops;

        return 0;

err_unset_ops:
        rtnl_lock();
        udp_tunnel_nic_ops = NULL;
        rtnl_unlock();
        destroy_workqueue(udp_tunnel_nic_workqueue);
        return err;
}
late_initcall(udp_tunnel_nic_init_module);

static void __exit udp_tunnel_nic_cleanup_module(void)
{
        unregister_netdevice_notifier(&udp_tunnel_nic_notifier_block);

        rtnl_lock();
        udp_tunnel_nic_ops = NULL;
        rtnl_unlock();

        destroy_workqueue(udp_tunnel_nic_workqueue);
}
module_exit(udp_tunnel_nic_cleanup_module);

MODULE_LICENSE("GPL");


































   37 









   45 





    7 



    7 





    7 
    7 








   16 



   16 
    3 
















   28 
   28 






   35 
















   35 






   35 



























    1 









    1 







    1 



    1 


























    2 
    2 









    1 

    1 








   12 














   12 

   12 


   12 




   12 








   52 






   45 


    7 
    7 





    2 


    7 





    2 






    2 
    1 

    1 









    2 






    3 


    2 




    1 



    1 










    2 
    1 



    1 













1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        Spanning tree protocol; interface code
 *        Linux ethernet bridge
 *
 *        Authors:
 *        Lennert Buytenhek                <buytenh@gnu.org>
 */

#include <linux/kernel.h>
#include <linux/kmod.h>
#include <linux/etherdevice.h>
#include <linux/rtnetlink.h>
#include <net/switchdev.h>

#include "br_private.h"
#include "br_private_stp.h"


/* Port id is composed of priority and port number.
 * NB: some bits of priority are dropped to
 *     make room for more ports.
 */
static inline port_id br_make_port_id(__u8 priority, __u16 port_no)
{
        return ((u16)priority << BR_PORT_BITS)
                | (port_no & ((1<<BR_PORT_BITS)-1));
}

#define BR_MAX_PORT_PRIORITY ((u16)~0 >> BR_PORT_BITS)

/* called under bridge lock */
void br_init_port(struct net_bridge_port *p)
{
        int err;

        p->port_id = br_make_port_id(p->priority, p->port_no);
        br_become_designated_port(p);
        br_set_state(p, BR_STATE_BLOCKING);
        p->topology_change_ack = 0;
        p->config_pending = 0;

        err = __set_ageing_time(p->dev, p->br->ageing_time);
        if (err)
                netdev_err(p->dev, "failed to offload ageing time\n");
}

/* NO locks held */
void br_stp_enable_bridge(struct net_bridge *br)
{
        struct net_bridge_port *p;

        spin_lock_bh(&br->lock);
        if (br->stp_enabled == BR_KERNEL_STP)
                mod_timer(&br->hello_timer, jiffies + br->hello_time);
        mod_delayed_work(system_long_wq, &br->gc_work, HZ / 10);

        br_config_bpdu_generation(br);

        list_for_each_entry(p, &br->port_list, list) {
                if (netif_running(p->dev) && netif_oper_up(p->dev))
                        br_stp_enable_port(p);

        }
        spin_unlock_bh(&br->lock);
}

/* NO locks held */
void br_stp_disable_bridge(struct net_bridge *br)
{
        struct net_bridge_port *p;

        spin_lock_bh(&br->lock);
        list_for_each_entry(p, &br->port_list, list) {
                if (p->state != BR_STATE_DISABLED)
                        br_stp_disable_port(p);

        }

        __br_set_topology_change(br, 0);
        br->topology_change_detected = 0;
        spin_unlock_bh(&br->lock);

        timer_delete_sync(&br->hello_timer);
        timer_delete_sync(&br->topology_change_timer);
        timer_delete_sync(&br->tcn_timer);
        cancel_delayed_work_sync(&br->gc_work);
}

/* called under bridge lock */
void br_stp_enable_port(struct net_bridge_port *p)
{
        br_init_port(p);
        br_port_state_selection(p->br);
        br_ifinfo_notify(RTM_NEWLINK, NULL, p);
}

/* called under bridge lock */
void br_stp_disable_port(struct net_bridge_port *p)
{
        struct net_bridge *br = p->br;
        int wasroot;

        wasroot = br_is_root_bridge(br);
        br_become_designated_port(p);
        br_set_state(p, BR_STATE_DISABLED);
        p->topology_change_ack = 0;
        p->config_pending = 0;

        br_ifinfo_notify(RTM_NEWLINK, NULL, p);

        timer_delete(&p->message_age_timer);
        timer_delete(&p->forward_delay_timer);
        timer_delete(&p->hold_timer);

        if (!rcu_access_pointer(p->backup_port))
                br_fdb_delete_by_port(br, p, 0, 0);
        br_multicast_disable_port(p);

        br_configuration_update(br);

        br_port_state_selection(br);

        if (br_is_root_bridge(br) && !wasroot)
                br_become_root_bridge(br);
}

static int br_stp_call_user(struct net_bridge *br, char *arg)
{
        char *argv[] = { BR_STP_PROG, br->dev->name, arg, NULL };
        char *envp[] = { NULL };
        int rc;

        /* call userspace STP and report program errors */
        rc = call_usermodehelper(BR_STP_PROG, argv, envp, UMH_WAIT_PROC);
        if (rc > 0) {
                if (rc & 0xff)
                        br_debug(br, BR_STP_PROG " received signal %d\n",
                                 rc & 0x7f);
                else
                        br_debug(br, BR_STP_PROG " exited with code %d\n",
                                 (rc >> 8) & 0xff);
        }

        return rc;
}

static void br_stp_start(struct net_bridge *br)
{
        int err = -ENOENT;

        if (net_eq(dev_net(br->dev), &init_net))
                err = br_stp_call_user(br, "start");

        if (err && err != -ENOENT)
                br_err(br, "failed to start userspace STP (%d)\n", err);

        spin_lock_bh(&br->lock);

        if (br->bridge_forward_delay < BR_MIN_FORWARD_DELAY)
                __br_set_forward_delay(br, BR_MIN_FORWARD_DELAY);
        else if (br->bridge_forward_delay > BR_MAX_FORWARD_DELAY)
                __br_set_forward_delay(br, BR_MAX_FORWARD_DELAY);

        if (!err) {
                br->stp_enabled = BR_USER_STP;
                br_debug(br, "userspace STP started\n");
        } else {
                br->stp_enabled = BR_KERNEL_STP;
                br_debug(br, "using kernel STP\n");

                /* To start timers on any ports left in blocking */
                if (br->dev->flags & IFF_UP)
                        mod_timer(&br->hello_timer, jiffies + br->hello_time);
                br_port_state_selection(br);
        }

        spin_unlock_bh(&br->lock);
}

static void br_stp_stop(struct net_bridge *br)
{
        int err;

        if (br->stp_enabled == BR_USER_STP) {
                err = br_stp_call_user(br, "stop");
                if (err)
                        br_err(br, "failed to stop userspace STP (%d)\n", err);

                /* To start timers on any ports left in blocking */
                spin_lock_bh(&br->lock);
                br_port_state_selection(br);
                spin_unlock_bh(&br->lock);
        }

        br->stp_enabled = BR_NO_STP;
}

int br_stp_set_enabled(struct net_bridge *br, unsigned long val,
                       struct netlink_ext_ack *extack)
{
        ASSERT_RTNL();

        if (br_mrp_enabled(br)) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "STP can't be enabled if MRP is already enabled");
                return -EINVAL;
        }

        if (val) {
                if (br->stp_enabled == BR_NO_STP)
                        br_stp_start(br);
        } else {
                if (br->stp_enabled != BR_NO_STP)
                        br_stp_stop(br);
        }

        return 0;
}

/* called under bridge lock */
void br_stp_change_bridge_id(struct net_bridge *br, const unsigned char *addr)
{
        /* should be aligned on 2 bytes for ether_addr_equal() */
        unsigned short oldaddr_aligned[ETH_ALEN >> 1];
        unsigned char *oldaddr = (unsigned char *)oldaddr_aligned;
        struct net_bridge_port *p;
        int wasroot;

        wasroot = br_is_root_bridge(br);

        br_fdb_change_mac_address(br, addr);

        memcpy(oldaddr, br->bridge_id.addr, ETH_ALEN);
        memcpy(br->bridge_id.addr, addr, ETH_ALEN);
        eth_hw_addr_set(br->dev, addr);

        list_for_each_entry(p, &br->port_list, list) {
                if (ether_addr_equal(p->designated_bridge.addr, oldaddr))
                        memcpy(p->designated_bridge.addr, addr, ETH_ALEN);

                if (ether_addr_equal(p->designated_root.addr, oldaddr))
                        memcpy(p->designated_root.addr, addr, ETH_ALEN);
        }

        br_configuration_update(br);
        br_port_state_selection(br);
        if (br_is_root_bridge(br) && !wasroot)
                br_become_root_bridge(br);
}

/* should be aligned on 2 bytes for ether_addr_equal() */
static const unsigned short br_mac_zero_aligned[ETH_ALEN >> 1];

/* called under bridge lock */
bool br_stp_recalculate_bridge_id(struct net_bridge *br)
{
        const unsigned char *br_mac_zero =
                        (const unsigned char *)br_mac_zero_aligned;
        const unsigned char *addr = br_mac_zero;
        struct net_bridge_port *p;

        /* user has chosen a value so keep it */
        if (br->dev->addr_assign_type == NET_ADDR_SET)
                return false;

        list_for_each_entry(p, &br->port_list, list) {
                if (addr == br_mac_zero ||
                    memcmp(p->dev->dev_addr, addr, ETH_ALEN) < 0)
                        addr = p->dev->dev_addr;

        }

        if (ether_addr_equal(br->bridge_id.addr, addr))
                return false;        /* no change */

        br_stp_change_bridge_id(br, addr);
        return true;
}

/* Acquires and releases bridge lock */
void br_stp_set_bridge_priority(struct net_bridge *br, u16 newprio)
{
        struct net_bridge_port *p;
        int wasroot;

        spin_lock_bh(&br->lock);
        wasroot = br_is_root_bridge(br);

        list_for_each_entry(p, &br->port_list, list) {
                if (p->state != BR_STATE_DISABLED &&
                    br_is_designated_port(p)) {
                        p->designated_bridge.prio[0] = (newprio >> 8) & 0xFF;
                        p->designated_bridge.prio[1] = newprio & 0xFF;
                }

        }

        br->bridge_id.prio[0] = (newprio >> 8) & 0xFF;
        br->bridge_id.prio[1] = newprio & 0xFF;
        br_configuration_update(br);
        br_port_state_selection(br);
        if (br_is_root_bridge(br) && !wasroot)
                br_become_root_bridge(br);
        spin_unlock_bh(&br->lock);
}

/* called under bridge lock */
int br_stp_set_port_priority(struct net_bridge_port *p, unsigned long newprio)
{
        port_id new_port_id;

        if (newprio > BR_MAX_PORT_PRIORITY)
                return -ERANGE;

        new_port_id = br_make_port_id(newprio, p->port_no);
        if (br_is_designated_port(p))
                p->designated_port = new_port_id;

        p->port_id = new_port_id;
        p->priority = newprio;
        if (!memcmp(&p->br->bridge_id, &p->designated_bridge, 8) &&
            p->port_id < p->designated_port) {
                br_become_designated_port(p);
                br_port_state_selection(p->br);
        }

        return 0;
}

/* called under bridge lock */
int br_stp_set_path_cost(struct net_bridge_port *p, unsigned long path_cost)
{
        if (path_cost < BR_MIN_PATH_COST ||
            path_cost > BR_MAX_PATH_COST)
                return -ERANGE;

        p->flags |= BR_ADMIN_COST;
        p->path_cost = path_cost;
        br_configuration_update(p->br);
        br_port_state_selection(p->br);
        return 0;
}

ssize_t br_show_bridge_id(char *buf, const struct bridge_id *id)
{
        return sprintf(buf, "%.2x%.2x.%.2x%.2x%.2x%.2x%.2x%.2x\n",
               id->prio[0], id->prio[1],
               id->addr[0], id->addr[1], id->addr[2],
               id->addr[3], id->addr[4], id->addr[5]);
}









































    1 



    1 












































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * sctp_offload - GRO/GSO Offloading for SCTP
 *
 * Copyright (C) 2015, Marcelo Ricardo Leitner <marcelo.leitner@gmail.com>
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/kernel.h>
#include <linux/kprobes.h>
#include <linux/socket.h>
#include <linux/sctp.h>
#include <linux/proc_fs.h>
#include <linux/vmalloc.h>
#include <linux/module.h>
#include <linux/kfifo.h>
#include <linux/time.h>
#include <net/net_namespace.h>

#include <linux/skbuff.h>
#include <net/sctp/sctp.h>
#include <net/sctp/checksum.h>
#include <net/protocol.h>
#include <net/gso.h>

static __le32 sctp_gso_make_checksum(struct sk_buff *skb)
{
        skb->ip_summed = CHECKSUM_NONE;
        skb->csum_not_inet = 0;
        /* csum and csum_start in GSO CB may be needed to do the UDP
         * checksum when it's a UDP tunneling packet.
         */
        SKB_GSO_CB(skb)->csum = (__force __wsum)~0;
        SKB_GSO_CB(skb)->csum_start = skb_headroom(skb) + skb->len;
        return sctp_compute_cksum(skb, skb_transport_offset(skb));
}

static struct sk_buff *sctp_gso_segment(struct sk_buff *skb,
                                        netdev_features_t features)
{
        struct sk_buff *segs = ERR_PTR(-EINVAL);
        struct sctphdr *sh;

        if (!skb_is_gso_sctp(skb))
                goto out;

        sh = sctp_hdr(skb);
        if (!pskb_may_pull(skb, sizeof(*sh)))
                goto out;

        __skb_pull(skb, sizeof(*sh));

        if (skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) {
                /* Packet is from an untrusted source, reset gso_segs. */
                struct skb_shared_info *pinfo = skb_shinfo(skb);
                struct sk_buff *frag_iter;

                pinfo->gso_segs = 0;
                if (skb->len != skb->data_len) {
                        /* Means we have chunks in here too */
                        pinfo->gso_segs++;
                }

                skb_walk_frags(skb, frag_iter)
                        pinfo->gso_segs++;

                segs = NULL;
                goto out;
        }

        segs = skb_segment(skb, (features | NETIF_F_HW_CSUM) & ~NETIF_F_SG);
        if (IS_ERR(segs))
                goto out;

        /* All that is left is update SCTP CRC if necessary */
        if (!(features & NETIF_F_SCTP_CRC)) {
                for (skb = segs; skb; skb = skb->next) {
                        if (skb->ip_summed == CHECKSUM_PARTIAL) {
                                sh = sctp_hdr(skb);
                                sh->checksum = sctp_gso_make_checksum(skb);
                        }
                }
        }

out:
        return segs;
}

static const struct net_offload sctp_offload = {
        .callbacks = {
                .gso_segment = sctp_gso_segment,
        },
};

static const struct net_offload sctp6_offload = {
        .callbacks = {
                .gso_segment = sctp_gso_segment,
        },
};

int __init sctp_offload_init(void)
{
        int ret;

        ret = inet_add_offload(&sctp_offload, IPPROTO_SCTP);
        if (ret)
                goto out;

        ret = inet6_add_offload(&sctp6_offload, IPPROTO_SCTP);
        if (ret)
                goto ipv4;

        crc32c_csum_stub = &sctp_csum_ops;
        return ret;

ipv4:
        inet_del_offload(&sctp_offload, IPPROTO_SCTP);
out:
        return ret;
}



































































































































   10 





   10 









   10 



















   10 
   10 




   10 






   10 




















































































































































































































































































































































































































































































































































































































































































   10 











































   10 









   10 







   10 







   10 





   10 












































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
// SPDX-License-Identifier: GPL-2.0
/*
 *  linux/fs/ext4/file.c
 *
 * Copyright (C) 1992, 1993, 1994, 1995
 * Remy Card (card@masi.ibp.fr)
 * Laboratoire MASI - Institut Blaise Pascal
 * Universite Pierre et Marie Curie (Paris VI)
 *
 *  from
 *
 *  linux/fs/minix/file.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *
 *  ext4 fs regular file handling primitives
 *
 *  64-bit file support on 64-bit platforms by Jakub Jelinek
 *        (jj@sunsite.ms.mff.cuni.cz)
 */

#include <linux/time.h>
#include <linux/fs.h>
#include <linux/iomap.h>
#include <linux/mount.h>
#include <linux/path.h>
#include <linux/dax.h>
#include <linux/quotaops.h>
#include <linux/pagevec.h>
#include <linux/uio.h>
#include <linux/mman.h>
#include <linux/backing-dev.h>
#include "ext4.h"
#include "ext4_jbd2.h"
#include "xattr.h"
#include "acl.h"
#include "truncate.h"

/*
 * Returns %true if the given DIO request should be attempted with DIO, or
 * %false if it should fall back to buffered I/O.
 *
 * DIO isn't well specified; when it's unsupported (either due to the request
 * being misaligned, or due to the file not supporting DIO at all), filesystems
 * either fall back to buffered I/O or return EINVAL.  For files that don't use
 * any special features like encryption or verity, ext4 has traditionally
 * returned EINVAL for misaligned DIO.  iomap_dio_rw() uses this convention too.
 * In this case, we should attempt the DIO, *not* fall back to buffered I/O.
 *
 * In contrast, in cases where DIO is unsupported due to ext4 features, ext4
 * traditionally falls back to buffered I/O.
 *
 * This function implements the traditional ext4 behavior in all these cases.
 */
static bool ext4_should_use_dio(struct kiocb *iocb, struct iov_iter *iter)
{
        struct inode *inode = file_inode(iocb->ki_filp);
        u32 dio_align = ext4_dio_alignment(inode);

        if (dio_align == 0)
                return false;

        if (dio_align == 1)
                return true;

        return IS_ALIGNED(iocb->ki_pos | iov_iter_alignment(iter), dio_align);
}

static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to)
{
        ssize_t ret;
        struct inode *inode = file_inode(iocb->ki_filp);

        if (iocb->ki_flags & IOCB_NOWAIT) {
                if (!inode_trylock_shared(inode))
                        return -EAGAIN;
        } else {
                inode_lock_shared(inode);
        }

        if (!ext4_should_use_dio(iocb, to)) {
                inode_unlock_shared(inode);
                /*
                 * Fallback to buffered I/O if the operation being performed on
                 * the inode is not supported by direct I/O. The IOCB_DIRECT
                 * flag needs to be cleared here in order to ensure that the
                 * direct I/O path within generic_file_read_iter() is not
                 * taken.
                 */
                iocb->ki_flags &= ~IOCB_DIRECT;
                return generic_file_read_iter(iocb, to);
        }

        ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0, NULL, 0);
        inode_unlock_shared(inode);

        file_accessed(iocb->ki_filp);
        return ret;
}

#ifdef CONFIG_FS_DAX
static ssize_t ext4_dax_read_iter(struct kiocb *iocb, struct iov_iter *to)
{
        struct inode *inode = file_inode(iocb->ki_filp);
        ssize_t ret;

        if (iocb->ki_flags & IOCB_NOWAIT) {
                if (!inode_trylock_shared(inode))
                        return -EAGAIN;
        } else {
                inode_lock_shared(inode);
        }
        /*
         * Recheck under inode lock - at this point we are sure it cannot
         * change anymore
         */
        if (!IS_DAX(inode)) {
                inode_unlock_shared(inode);
                /* Fallback to buffered IO in case we cannot support DAX */
                return generic_file_read_iter(iocb, to);
        }
        ret = dax_iomap_rw(iocb, to, &ext4_iomap_ops);
        inode_unlock_shared(inode);

        file_accessed(iocb->ki_filp);
        return ret;
}
#endif

static ssize_t ext4_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
{
        struct inode *inode = file_inode(iocb->ki_filp);

        if (unlikely(ext4_forced_shutdown(inode->i_sb)))
                return -EIO;

        if (!iov_iter_count(to))
                return 0; /* skip atime */

#ifdef CONFIG_FS_DAX
        if (IS_DAX(inode))
                return ext4_dax_read_iter(iocb, to);
#endif
        if (iocb->ki_flags & IOCB_DIRECT)
                return ext4_dio_read_iter(iocb, to);

        return generic_file_read_iter(iocb, to);
}

static ssize_t ext4_file_splice_read(struct file *in, loff_t *ppos,
                                     struct pipe_inode_info *pipe,
                                     size_t len, unsigned int flags)
{
        struct inode *inode = file_inode(in);

        if (unlikely(ext4_forced_shutdown(inode->i_sb)))
                return -EIO;
        return filemap_splice_read(in, ppos, pipe, len, flags);
}

/*
 * Called when an inode is released. Note that this is different
 * from ext4_file_open: open gets called at every open, but release
 * gets called only when /all/ the files are closed.
 */
static int ext4_release_file(struct inode *inode, struct file *filp)
{
        if (ext4_test_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE)) {
                ext4_alloc_da_blocks(inode);
                ext4_clear_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE);
        }
        /* if we are the last writer on the inode, drop the block reservation */
        if ((filp->f_mode & FMODE_WRITE) &&
                        (atomic_read(&inode->i_writecount) == 1) &&
                        !EXT4_I(inode)->i_reserved_data_blocks) {
                down_write(&EXT4_I(inode)->i_data_sem);
                ext4_discard_preallocations(inode);
                up_write(&EXT4_I(inode)->i_data_sem);
        }
        if (is_dx(inode) && filp->private_data)
                ext4_htree_free_dir_info(filp->private_data);

        return 0;
}

/*
 * This tests whether the IO in question is block-aligned or not.
 * Ext4 utilizes unwritten extents when hole-filling during direct IO, and they
 * are converted to written only after the IO is complete.  Until they are
 * mapped, these blocks appear as holes, so dio_zero_block() will assume that
 * it needs to zero out portions of the start and/or end block.  If 2 AIO
 * threads are at work on the same unwritten block, they must be synchronized
 * or one thread will zero the other's data, causing corruption.
 */
static bool
ext4_unaligned_io(struct inode *inode, struct iov_iter *from, loff_t pos)
{
        struct super_block *sb = inode->i_sb;
        unsigned long blockmask = sb->s_blocksize - 1;

        if ((pos | iov_iter_alignment(from)) & blockmask)
                return true;

        return false;
}

static bool
ext4_extending_io(struct inode *inode, loff_t offset, size_t len)
{
        if (offset + len > i_size_read(inode) ||
            offset + len > EXT4_I(inode)->i_disksize)
                return true;
        return false;
}

/* Is IO overwriting allocated or initialized blocks? */
static bool ext4_overwrite_io(struct inode *inode,
                              loff_t pos, loff_t len, bool *unwritten)
{
        struct ext4_map_blocks map;
        unsigned int blkbits = inode->i_blkbits;
        int err, blklen;

        if (pos + len > i_size_read(inode))
                return false;

        map.m_lblk = pos >> blkbits;
        map.m_len = EXT4_MAX_BLOCKS(len, pos, blkbits);
        blklen = map.m_len;

        err = ext4_map_blocks(NULL, inode, &map, 0);
        if (err != blklen)
                return false;
        /*
         * 'err==len' means that all of the blocks have been preallocated,
         * regardless of whether they have been initialized or not. We need to
         * check m_flags to distinguish the unwritten extents.
         */
        *unwritten = !(map.m_flags & EXT4_MAP_MAPPED);
        return true;
}

static ssize_t ext4_generic_write_checks(struct kiocb *iocb,
                                         struct iov_iter *from)
{
        struct inode *inode = file_inode(iocb->ki_filp);
        ssize_t ret;

        if (unlikely(IS_IMMUTABLE(inode)))
                return -EPERM;

        ret = generic_write_checks(iocb, from);
        if (ret <= 0)
                return ret;

        /*
         * If we have encountered a bitmap-format file, the size limit
         * is smaller than s_maxbytes, which is for extent-mapped files.
         */
        if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
                struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);

                if (iocb->ki_pos >= sbi->s_bitmap_maxbytes)
                        return -EFBIG;
                iov_iter_truncate(from, sbi->s_bitmap_maxbytes - iocb->ki_pos);
        }

        return iov_iter_count(from);
}

static ssize_t ext4_write_checks(struct kiocb *iocb, struct iov_iter *from)
{
        ssize_t ret, count;

        count = ext4_generic_write_checks(iocb, from);
        if (count <= 0)
                return count;

        ret = file_modified(iocb->ki_filp);
        if (ret)
                return ret;
        return count;
}

static ssize_t ext4_buffered_write_iter(struct kiocb *iocb,
                                        struct iov_iter *from)
{
        ssize_t ret;
        struct inode *inode = file_inode(iocb->ki_filp);

        if (iocb->ki_flags & IOCB_NOWAIT)
                return -EOPNOTSUPP;

        inode_lock(inode);
        ret = ext4_write_checks(iocb, from);
        if (ret <= 0)
                goto out;

        ret = generic_perform_write(iocb, from);

out:
        inode_unlock(inode);
        if (unlikely(ret <= 0))
                return ret;
        return generic_write_sync(iocb, ret);
}

static ssize_t ext4_handle_inode_extension(struct inode *inode, loff_t offset,
                                           ssize_t written, ssize_t count)
{
        handle_t *handle;

        lockdep_assert_held_write(&inode->i_rwsem);
        handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
        if (IS_ERR(handle))
                return PTR_ERR(handle);

        if (ext4_update_inode_size(inode, offset + written)) {
                int ret = ext4_mark_inode_dirty(handle, inode);
                if (unlikely(ret)) {
                        ext4_journal_stop(handle);
                        return ret;
                }
        }

        if ((written == count) && inode->i_nlink)
                ext4_orphan_del(handle, inode);
        ext4_journal_stop(handle);

        return written;
}

/*
 * Clean up the inode after DIO or DAX extending write has completed and the
 * inode size has been updated using ext4_handle_inode_extension().
 */
static void ext4_inode_extension_cleanup(struct inode *inode, bool need_trunc)
{
        lockdep_assert_held_write(&inode->i_rwsem);
        if (need_trunc) {
                ext4_truncate_failed_write(inode);
                /*
                 * If the truncate operation failed early, then the inode may
                 * still be on the orphan list. In that case, we need to try
                 * remove the inode from the in-memory linked list.
                 */
                if (inode->i_nlink)
                        ext4_orphan_del(NULL, inode);
                return;
        }
        /*
         * If i_disksize got extended either due to writeback of delalloc
         * blocks or extending truncate while the DIO was running we could fail
         * to cleanup the orphan list in ext4_handle_inode_extension(). Do it
         * now.
         */
        if (!list_empty(&EXT4_I(inode)->i_orphan) && inode->i_nlink) {
                handle_t *handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);

                if (IS_ERR(handle)) {
                        /*
                         * The write has successfully completed. Not much to
                         * do with the error here so just cleanup the orphan
                         * list and hope for the best.
                         */
                        ext4_orphan_del(NULL, inode);
                        return;
                }
                ext4_orphan_del(handle, inode);
                ext4_journal_stop(handle);
        }
}

static int ext4_dio_write_end_io(struct kiocb *iocb, ssize_t size,
                                 int error, unsigned int flags)
{
        loff_t pos = iocb->ki_pos;
        struct inode *inode = file_inode(iocb->ki_filp);

        if (!error && size && flags & IOMAP_DIO_UNWRITTEN)
                error = ext4_convert_unwritten_extents(NULL, inode, pos, size);
        if (error)
                return error;
        /*
         * Note that EXT4_I(inode)->i_disksize can get extended up to
         * inode->i_size while the I/O was running due to writeback of delalloc
         * blocks. But the code in ext4_iomap_alloc() is careful to use
         * zeroed/unwritten extents if this is possible; thus we won't leave
         * uninitialized blocks in a file even if we didn't succeed in writing
         * as much as we intended. Also we can race with truncate or write
         * expanding the file so we have to be a bit careful here.
         */
        if (pos + size <= READ_ONCE(EXT4_I(inode)->i_disksize) &&
            pos + size <= i_size_read(inode))
                return 0;
        error = ext4_handle_inode_extension(inode, pos, size, size);
        return error < 0 ? error : 0;
}

static const struct iomap_dio_ops ext4_dio_write_ops = {
        .end_io = ext4_dio_write_end_io,
};

/*
 * The intention here is to start with shared lock acquired then see if any
 * condition requires an exclusive inode lock. If yes, then we restart the
 * whole operation by releasing the shared lock and acquiring exclusive lock.
 *
 * - For unaligned_io we never take shared lock as it may cause data corruption
 *   when two unaligned IO tries to modify the same block e.g. while zeroing.
 *
 * - For extending writes case we don't take the shared lock, since it requires
 *   updating inode i_disksize and/or orphan handling with exclusive lock.
 *
 * - shared locking will only be true mostly with overwrites, including
 *   initialized blocks and unwritten blocks. For overwrite unwritten blocks
 *   we protect splitting extents by i_data_sem in ext4_inode_info, so we can
 *   also release exclusive i_rwsem lock.
 *
 * - Otherwise we will switch to exclusive i_rwsem lock.
 */
static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from,
                                     bool *ilock_shared, bool *extend,
                                     bool *unwritten, int *dio_flags)
{
        struct file *file = iocb->ki_filp;
        struct inode *inode = file_inode(file);
        loff_t offset;
        size_t count;
        ssize_t ret;
        bool overwrite, unaligned_io;

restart:
        ret = ext4_generic_write_checks(iocb, from);
        if (ret <= 0)
                goto out;

        offset = iocb->ki_pos;
        count = ret;

        unaligned_io = ext4_unaligned_io(inode, from, offset);
        *extend = ext4_extending_io(inode, offset, count);
        overwrite = ext4_overwrite_io(inode, offset, count, unwritten);

        /*
         * Determine whether we need to upgrade to an exclusive lock. This is
         * required to change security info in file_modified(), for extending
         * I/O, any form of non-overwrite I/O, and unaligned I/O to unwritten
         * extents (as partial block zeroing may be required).
         *
         * Note that unaligned writes are allowed under shared lock so long as
         * they are pure overwrites. Otherwise, concurrent unaligned writes risk
         * data corruption due to partial block zeroing in the dio layer, and so
         * the I/O must occur exclusively.
         */
        if (*ilock_shared &&
            ((!IS_NOSEC(inode) || *extend || !overwrite ||
             (unaligned_io && *unwritten)))) {
                if (iocb->ki_flags & IOCB_NOWAIT) {
                        ret = -EAGAIN;
                        goto out;
                }
                inode_unlock_shared(inode);
                *ilock_shared = false;
                inode_lock(inode);
                goto restart;
        }

        /*
         * Now that locking is settled, determine dio flags and exclusivity
         * requirements. We don't use DIO_OVERWRITE_ONLY because we enforce
         * behavior already. The inode lock is already held exclusive if the
         * write is non-overwrite or extending, so drain all outstanding dio and
         * set the force wait dio flag.
         */
        if (!*ilock_shared && (unaligned_io || *extend)) {
                if (iocb->ki_flags & IOCB_NOWAIT) {
                        ret = -EAGAIN;
                        goto out;
                }
                if (unaligned_io && (!overwrite || *unwritten))
                        inode_dio_wait(inode);
                *dio_flags = IOMAP_DIO_FORCE_WAIT;
        }

        ret = file_modified(file);
        if (ret < 0)
                goto out;

        return count;
out:
        if (*ilock_shared)
                inode_unlock_shared(inode);
        else
                inode_unlock(inode);
        return ret;
}

static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from)
{
        ssize_t ret;
        handle_t *handle;
        struct inode *inode = file_inode(iocb->ki_filp);
        loff_t offset = iocb->ki_pos;
        size_t count = iov_iter_count(from);
        const struct iomap_ops *iomap_ops = &ext4_iomap_ops;
        bool extend = false, unwritten = false;
        bool ilock_shared = true;
        int dio_flags = 0;

        /*
         * Quick check here without any i_rwsem lock to see if it is extending
         * IO. A more reliable check is done in ext4_dio_write_checks() with
         * proper locking in place.
         */
        if (offset + count > i_size_read(inode))
                ilock_shared = false;

        if (iocb->ki_flags & IOCB_NOWAIT) {
                if (ilock_shared) {
                        if (!inode_trylock_shared(inode))
                                return -EAGAIN;
                } else {
                        if (!inode_trylock(inode))
                                return -EAGAIN;
                }
        } else {
                if (ilock_shared)
                        inode_lock_shared(inode);
                else
                        inode_lock(inode);
        }

        /* Fallback to buffered I/O if the inode does not support direct I/O. */
        if (!ext4_should_use_dio(iocb, from)) {
                if (ilock_shared)
                        inode_unlock_shared(inode);
                else
                        inode_unlock(inode);
                return ext4_buffered_write_iter(iocb, from);
        }

        /*
         * Prevent inline data from being created since we are going to allocate
         * blocks for DIO. We know the inode does not currently have inline data
         * because ext4_should_use_dio() checked for it, but we have to clear
         * the state flag before the write checks because a lock cycle could
         * introduce races with other writers.
         */
        ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);

        ret = ext4_dio_write_checks(iocb, from, &ilock_shared, &extend,
                                    &unwritten, &dio_flags);
        if (ret <= 0)
                return ret;

        offset = iocb->ki_pos;
        count = ret;

        if (extend) {
                handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
                if (IS_ERR(handle)) {
                        ret = PTR_ERR(handle);
                        goto out;
                }

                ret = ext4_orphan_add(handle, inode);
                ext4_journal_stop(handle);
                if (ret)
                        goto out;
        }

        if (ilock_shared && !unwritten)
                iomap_ops = &ext4_iomap_overwrite_ops;
        ret = iomap_dio_rw(iocb, from, iomap_ops, &ext4_dio_write_ops,
                           dio_flags, NULL, 0);
        if (ret == -ENOTBLK)
                ret = 0;
        if (extend) {
                /*
                 * We always perform extending DIO write synchronously so by
                 * now the IO is completed and ext4_handle_inode_extension()
                 * was called. Cleanup the inode in case of error or race with
                 * writeback of delalloc blocks.
                 */
                WARN_ON_ONCE(ret == -EIOCBQUEUED);
                ext4_inode_extension_cleanup(inode, ret < 0);
        }

out:
        if (ilock_shared)
                inode_unlock_shared(inode);
        else
                inode_unlock(inode);

        if (ret >= 0 && iov_iter_count(from)) {
                ssize_t err;
                loff_t endbyte;

                /*
                 * There is no support for atomic writes on buffered-io yet,
                 * we should never fallback to buffered-io for DIO atomic
                 * writes.
                 */
                WARN_ON_ONCE(iocb->ki_flags & IOCB_ATOMIC);

                offset = iocb->ki_pos;
                err = ext4_buffered_write_iter(iocb, from);
                if (err < 0)
                        return err;

                /*
                 * We need to ensure that the pages within the page cache for
                 * the range covered by this I/O are written to disk and
                 * invalidated. This is in attempt to preserve the expected
                 * direct I/O semantics in the case we fallback to buffered I/O
                 * to complete off the I/O request.
                 */
                ret += err;
                endbyte = offset + err - 1;
                err = filemap_write_and_wait_range(iocb->ki_filp->f_mapping,
                                                   offset, endbyte);
                if (!err)
                        invalidate_mapping_pages(iocb->ki_filp->f_mapping,
                                                 offset >> PAGE_SHIFT,
                                                 endbyte >> PAGE_SHIFT);
        }

        return ret;
}

#ifdef CONFIG_FS_DAX
static ssize_t
ext4_dax_write_iter(struct kiocb *iocb, struct iov_iter *from)
{
        ssize_t ret;
        size_t count;
        loff_t offset;
        handle_t *handle;
        bool extend = false;
        struct inode *inode = file_inode(iocb->ki_filp);

        if (iocb->ki_flags & IOCB_NOWAIT) {
                if (!inode_trylock(inode))
                        return -EAGAIN;
        } else {
                inode_lock(inode);
        }

        ret = ext4_write_checks(iocb, from);
        if (ret <= 0)
                goto out;

        offset = iocb->ki_pos;
        count = iov_iter_count(from);

        if (offset + count > EXT4_I(inode)->i_disksize) {
                handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
                if (IS_ERR(handle)) {
                        ret = PTR_ERR(handle);
                        goto out;
                }

                ret = ext4_orphan_add(handle, inode);
                if (ret) {
                        ext4_journal_stop(handle);
                        goto out;
                }

                extend = true;
                ext4_journal_stop(handle);
        }

        ret = dax_iomap_rw(iocb, from, &ext4_iomap_ops);

        if (extend) {
                ret = ext4_handle_inode_extension(inode, offset, ret, count);
                ext4_inode_extension_cleanup(inode, ret < (ssize_t)count);
        }
out:
        inode_unlock(inode);
        if (ret > 0)
                ret = generic_write_sync(iocb, ret);
        return ret;
}
#endif

static ssize_t
ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
{
        int ret;
        struct inode *inode = file_inode(iocb->ki_filp);

        ret = ext4_emergency_state(inode->i_sb);
        if (unlikely(ret))
                return ret;

#ifdef CONFIG_FS_DAX
        if (IS_DAX(inode))
                return ext4_dax_write_iter(iocb, from);
#endif

        if (iocb->ki_flags & IOCB_ATOMIC) {
                size_t len = iov_iter_count(from);

                if (len < EXT4_SB(inode->i_sb)->s_awu_min ||
                    len > EXT4_SB(inode->i_sb)->s_awu_max)
                        return -EINVAL;

                ret = generic_atomic_write_valid(iocb, from);
                if (ret)
                        return ret;
        }

        if (iocb->ki_flags & IOCB_DIRECT)
                return ext4_dio_write_iter(iocb, from);
        else
                return ext4_buffered_write_iter(iocb, from);
}

#ifdef CONFIG_FS_DAX
static vm_fault_t ext4_dax_huge_fault(struct vm_fault *vmf, unsigned int order)
{
        int error = 0;
        vm_fault_t result;
        int retries = 0;
        handle_t *handle = NULL;
        struct inode *inode = file_inode(vmf->vma->vm_file);
        struct super_block *sb = inode->i_sb;

        /*
         * We have to distinguish real writes from writes which will result in a
         * COW page; COW writes should *not* poke the journal (the file will not
         * be changed). Doing so would cause unintended failures when mounted
         * read-only.
         *
         * We check for VM_SHARED rather than vmf->cow_page since the latter is
         * unset for order != 0 (i.e. only in do_cow_fault); for
         * other sizes, dax_iomap_fault will handle splitting / fallback so that
         * we eventually come back with a COW page.
         */
        bool write = (vmf->flags & FAULT_FLAG_WRITE) &&
                (vmf->vma->vm_flags & VM_SHARED);
        struct address_space *mapping = vmf->vma->vm_file->f_mapping;
        pfn_t pfn;

        if (write) {
                sb_start_pagefault(sb);
                file_update_time(vmf->vma->vm_file);
                filemap_invalidate_lock_shared(mapping);
retry:
                handle = ext4_journal_start_sb(sb, EXT4_HT_WRITE_PAGE,
                                               EXT4_DATA_TRANS_BLOCKS(sb));
                if (IS_ERR(handle)) {
                        filemap_invalidate_unlock_shared(mapping);
                        sb_end_pagefault(sb);
                        return VM_FAULT_SIGBUS;
                }
        } else {
                filemap_invalidate_lock_shared(mapping);
        }
        result = dax_iomap_fault(vmf, order, &pfn, &error, &ext4_iomap_ops);
        if (write) {
                ext4_journal_stop(handle);

                if ((result & VM_FAULT_ERROR) && error == -ENOSPC &&
                    ext4_should_retry_alloc(sb, &retries))
                        goto retry;
                /* Handling synchronous page fault? */
                if (result & VM_FAULT_NEEDDSYNC)
                        result = dax_finish_sync_fault(vmf, order, pfn);
                filemap_invalidate_unlock_shared(mapping);
                sb_end_pagefault(sb);
        } else {
                filemap_invalidate_unlock_shared(mapping);
        }

        return result;
}

static vm_fault_t ext4_dax_fault(struct vm_fault *vmf)
{
        return ext4_dax_huge_fault(vmf, 0);
}

static const struct vm_operations_struct ext4_dax_vm_ops = {
        .fault                = ext4_dax_fault,
        .huge_fault        = ext4_dax_huge_fault,
        .page_mkwrite        = ext4_dax_fault,
        .pfn_mkwrite        = ext4_dax_fault,
};
#else
#define ext4_dax_vm_ops        ext4_file_vm_ops
#endif

static const struct vm_operations_struct ext4_file_vm_ops = {
        .fault                = filemap_fault,
        .map_pages        = filemap_map_pages,
        .page_mkwrite   = ext4_page_mkwrite,
};

static int ext4_file_mmap(struct file *file, struct vm_area_struct *vma)
{
        int ret;
        struct inode *inode = file->f_mapping->host;
        struct dax_device *dax_dev = EXT4_SB(inode->i_sb)->s_daxdev;

        if (file->f_mode & FMODE_WRITE)
                ret = ext4_emergency_state(inode->i_sb);
        else
                ret = ext4_forced_shutdown(inode->i_sb) ? -EIO : 0;
        if (unlikely(ret))
                return ret;

        /*
         * We don't support synchronous mappings for non-DAX files and
         * for DAX files if underneath dax_device is not synchronous.
         */
        if (!daxdev_mapping_supported(vma, dax_dev))
                return -EOPNOTSUPP;

        file_accessed(file);
        if (IS_DAX(file_inode(file))) {
                vma->vm_ops = &ext4_dax_vm_ops;
                vm_flags_set(vma, VM_HUGEPAGE);
        } else {
                vma->vm_ops = &ext4_file_vm_ops;
        }
        return 0;
}

static int ext4_sample_last_mounted(struct super_block *sb,
                                    struct vfsmount *mnt)
{
        struct ext4_sb_info *sbi = EXT4_SB(sb);
        struct path path;
        char buf[64], *cp;
        handle_t *handle;
        int err;

        if (likely(ext4_test_mount_flag(sb, EXT4_MF_MNTDIR_SAMPLED)))
                return 0;

        if (ext4_emergency_state(sb) || sb_rdonly(sb) ||
            !sb_start_intwrite_trylock(sb))
                return 0;

        ext4_set_mount_flag(sb, EXT4_MF_MNTDIR_SAMPLED);
        /*
         * Sample where the filesystem has been mounted and
         * store it in the superblock for sysadmin convenience
         * when trying to sort through large numbers of block
         * devices or filesystem images.
         */
        memset(buf, 0, sizeof(buf));
        path.mnt = mnt;
        path.dentry = mnt->mnt_root;
        cp = d_path(&path, buf, sizeof(buf));
        err = 0;
        if (IS_ERR(cp))
                goto out;

        handle = ext4_journal_start_sb(sb, EXT4_HT_MISC, 1);
        err = PTR_ERR(handle);
        if (IS_ERR(handle))
                goto out;
        BUFFER_TRACE(sbi->s_sbh, "get_write_access");
        err = ext4_journal_get_write_access(handle, sb, sbi->s_sbh,
                                            EXT4_JTR_NONE);
        if (err)
                goto out_journal;
        lock_buffer(sbi->s_sbh);
        strtomem_pad(sbi->s_es->s_last_mounted, cp, 0);
        ext4_superblock_csum_set(sb);
        unlock_buffer(sbi->s_sbh);
        ext4_handle_dirty_metadata(handle, NULL, sbi->s_sbh);
out_journal:
        ext4_journal_stop(handle);
out:
        sb_end_intwrite(sb);
        return err;
}

static int ext4_file_open(struct inode *inode, struct file *filp)
{
        int ret;

        if (filp->f_mode & FMODE_WRITE)
                ret = ext4_emergency_state(inode->i_sb);
        else
                ret = ext4_forced_shutdown(inode->i_sb) ? -EIO : 0;
        if (unlikely(ret))
                return ret;

        ret = ext4_sample_last_mounted(inode->i_sb, filp->f_path.mnt);
        if (ret)
                return ret;

        ret = fscrypt_file_open(inode, filp);
        if (ret)
                return ret;

        ret = fsverity_file_open(inode, filp);
        if (ret)
                return ret;

        /*
         * Set up the jbd2_inode if we are opening the inode for
         * writing and the journal is present
         */
        if (filp->f_mode & FMODE_WRITE) {
                ret = ext4_inode_attach_jinode(inode);
                if (ret < 0)
                        return ret;
        }

        if (ext4_inode_can_atomic_write(inode))
                filp->f_mode |= FMODE_CAN_ATOMIC_WRITE;

        filp->f_mode |= FMODE_NOWAIT | FMODE_CAN_ODIRECT;
        return dquot_file_open(inode, filp);
}

/*
 * ext4_llseek() handles both block-mapped and extent-mapped maxbytes values
 * by calling generic_file_llseek_size() with the appropriate maxbytes
 * value for each.
 */
loff_t ext4_llseek(struct file *file, loff_t offset, int whence)
{
        struct inode *inode = file->f_mapping->host;
        loff_t maxbytes;

        if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)))
                maxbytes = EXT4_SB(inode->i_sb)->s_bitmap_maxbytes;
        else
                maxbytes = inode->i_sb->s_maxbytes;

        switch (whence) {
        default:
                return generic_file_llseek_size(file, offset, whence,
                                                maxbytes, i_size_read(inode));
        case SEEK_HOLE:
                inode_lock_shared(inode);
                offset = iomap_seek_hole(inode, offset,
                                         &ext4_iomap_report_ops);
                inode_unlock_shared(inode);
                break;
        case SEEK_DATA:
                inode_lock_shared(inode);
                offset = iomap_seek_data(inode, offset,
                                         &ext4_iomap_report_ops);
                inode_unlock_shared(inode);
                break;
        }

        if (offset < 0)
                return offset;
        return vfs_setpos(file, offset, maxbytes);
}

const struct file_operations ext4_file_operations = {
        .llseek                = ext4_llseek,
        .read_iter        = ext4_file_read_iter,
        .write_iter        = ext4_file_write_iter,
        .iopoll                = iocb_bio_iopoll,
        .unlocked_ioctl = ext4_ioctl,
#ifdef CONFIG_COMPAT
        .compat_ioctl        = ext4_compat_ioctl,
#endif
        .mmap                = ext4_file_mmap,
        .open                = ext4_file_open,
        .release        = ext4_release_file,
        .fsync                = ext4_sync_file,
        .get_unmapped_area = thp_get_unmapped_area,
        .splice_read        = ext4_file_splice_read,
        .splice_write        = iter_file_splice_write,
        .fallocate        = ext4_fallocate,
        .fop_flags        = FOP_MMAP_SYNC | FOP_BUFFER_RASYNC |
                          FOP_DIO_PARALLEL_WRITE,
};

const struct inode_operations ext4_file_inode_operations = {
        .setattr        = ext4_setattr,
        .getattr        = ext4_file_getattr,
        .listxattr        = ext4_listxattr,
        .get_inode_acl        = ext4_get_acl,
        .set_acl        = ext4_set_acl,
        .fiemap                = ext4_fiemap,
        .fileattr_get        = ext4_fileattr_get,
        .fileattr_set        = ext4_fileattr_set,
};










































































































































































































































































































































































 2301 








 2301 





 2301 


































 2294 



























 2291 
 2294 
 2292 







 2245 
































 2295 


  752 
 2298 
 2293 
   12 

















































 2296 













  744 






























  744 
  743 


























  744 

























 2290 




































 2301 




  745 










































































  744 

























   10 


























































   10 




















































 2298 

























 2297 





   10 















































 2293 






 2295 




































 2300 



    5 









 2297 





















 2300 

 2293 


























 2300 


 2301 

    5 









































   10 



   10 


   10 



   10 



   10 








   10 

   10 






























 2293 

 2295 


 2298 


 2298 






    5 
    5 














 2244 



 2243 
















 2243 

 2247 



    6 
    6 

    6 


















 2240 


























    3 


    8 








































































   12 






   12 


   10 

    3 





   10 
















   10 























   10 






   10 





   10 


   10 


























































 2244 
















































 2244 














 2297 

 2295 

   68 






























 2294 






 2300 








 2291 

 2301 






 2292 




























 2297 
 2291 

   10 








 2301 









 2291 













 2292 

 2296 



























 2299 





 2301 



















   70 











   68 



















 2241 






























































 2243 






 2246 































 2294 

   10 










 2251 
















 2293 










 2301 





 2244 




 2240 


 2251 





 2245 




 2285 















 2292 

















































































































































 2293 



 2301 
































 2288 





































 2286 












































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
// SPDX-License-Identifier: GPL-2.0

#include <linux/kernel.h>
#include <linux/irqflags.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/bug.h>
#include "printk_ringbuffer.h"
#include "internal.h"

/**
 * DOC: printk_ringbuffer overview
 *
 * Data Structure
 * --------------
 * The printk_ringbuffer is made up of 3 internal ringbuffers:
 *
 *   desc_ring
 *     A ring of descriptors and their meta data (such as sequence number,
 *     timestamp, loglevel, etc.) as well as internal state information about
 *     the record and logical positions specifying where in the other
 *     ringbuffer the text strings are located.
 *
 *   text_data_ring
 *     A ring of data blocks. A data block consists of an unsigned long
 *     integer (ID) that maps to a desc_ring index followed by the text
 *     string of the record.
 *
 * The internal state information of a descriptor is the key element to allow
 * readers and writers to locklessly synchronize access to the data.
 *
 * Implementation
 * --------------
 *
 * Descriptor Ring
 * ~~~~~~~~~~~~~~~
 * The descriptor ring is an array of descriptors. A descriptor contains
 * essential meta data to track the data of a printk record using
 * blk_lpos structs pointing to associated text data blocks (see
 * "Data Rings" below). Each descriptor is assigned an ID that maps
 * directly to index values of the descriptor array and has a state. The ID
 * and the state are bitwise combined into a single descriptor field named
 * @state_var, allowing ID and state to be synchronously and atomically
 * updated.
 *
 * Descriptors have four states:
 *
 *   reserved
 *     A writer is modifying the record.
 *
 *   committed
 *     The record and all its data are written. A writer can reopen the
 *     descriptor (transitioning it back to reserved), but in the committed
 *     state the data is consistent.
 *
 *   finalized
 *     The record and all its data are complete and available for reading. A
 *     writer cannot reopen the descriptor.
 *
 *   reusable
 *     The record exists, but its text and/or meta data may no longer be
 *     available.
 *
 * Querying the @state_var of a record requires providing the ID of the
 * descriptor to query. This can yield a possible fifth (pseudo) state:
 *
 *   miss
 *     The descriptor being queried has an unexpected ID.
 *
 * The descriptor ring has a @tail_id that contains the ID of the oldest
 * descriptor and @head_id that contains the ID of the newest descriptor.
 *
 * When a new descriptor should be created (and the ring is full), the tail
 * descriptor is invalidated by first transitioning to the reusable state and
 * then invalidating all tail data blocks up to and including the data blocks
 * associated with the tail descriptor (for the text ring). Then
 * @tail_id is advanced, followed by advancing @head_id. And finally the
 * @state_var of the new descriptor is initialized to the new ID and reserved
 * state.
 *
 * The @tail_id can only be advanced if the new @tail_id would be in the
 * committed or reusable queried state. This makes it possible that a valid
 * sequence number of the tail is always available.
 *
 * Descriptor Finalization
 * ~~~~~~~~~~~~~~~~~~~~~~~
 * When a writer calls the commit function prb_commit(), record data is
 * fully stored and is consistent within the ringbuffer. However, a writer can
 * reopen that record, claiming exclusive access (as with prb_reserve()), and
 * modify that record. When finished, the writer must again commit the record.
 *
 * In order for a record to be made available to readers (and also become
 * recyclable for writers), it must be finalized. A finalized record cannot be
 * reopened and can never become "unfinalized". Record finalization can occur
 * in three different scenarios:
 *
 *   1) A writer can simultaneously commit and finalize its record by calling
 *      prb_final_commit() instead of prb_commit().
 *
 *   2) When a new record is reserved and the previous record has been
 *      committed via prb_commit(), that previous record is automatically
 *      finalized.
 *
 *   3) When a record is committed via prb_commit() and a newer record
 *      already exists, the record being committed is automatically finalized.
 *
 * Data Ring
 * ~~~~~~~~~
 * The text data ring is a byte array composed of data blocks. Data blocks are
 * referenced by blk_lpos structs that point to the logical position of the
 * beginning of a data block and the beginning of the next adjacent data
 * block. Logical positions are mapped directly to index values of the byte
 * array ringbuffer.
 *
 * Each data block consists of an ID followed by the writer data. The ID is
 * the identifier of a descriptor that is associated with the data block. A
 * given data block is considered valid if all of the following conditions
 * are met:
 *
 *   1) The descriptor associated with the data block is in the committed
 *      or finalized queried state.
 *
 *   2) The blk_lpos struct within the descriptor associated with the data
 *      block references back to the same data block.
 *
 *   3) The data block is within the head/tail logical position range.
 *
 * If the writer data of a data block would extend beyond the end of the
 * byte array, only the ID of the data block is stored at the logical
 * position and the full data block (ID and writer data) is stored at the
 * beginning of the byte array. The referencing blk_lpos will point to the
 * ID before the wrap and the next data block will be at the logical
 * position adjacent the full data block after the wrap.
 *
 * Data rings have a @tail_lpos that points to the beginning of the oldest
 * data block and a @head_lpos that points to the logical position of the
 * next (not yet existing) data block.
 *
 * When a new data block should be created (and the ring is full), tail data
 * blocks will first be invalidated by putting their associated descriptors
 * into the reusable state and then pushing the @tail_lpos forward beyond
 * them. Then the @head_lpos is pushed forward and is associated with a new
 * descriptor. If a data block is not valid, the @tail_lpos cannot be
 * advanced beyond it.
 *
 * Info Array
 * ~~~~~~~~~~
 * The general meta data of printk records are stored in printk_info structs,
 * stored in an array with the same number of elements as the descriptor ring.
 * Each info corresponds to the descriptor of the same index in the
 * descriptor ring. Info validity is confirmed by evaluating the corresponding
 * descriptor before and after loading the info.
 *
 * Usage
 * -----
 * Here are some simple examples demonstrating writers and readers. For the
 * examples a global ringbuffer (test_rb) is available (which is not the
 * actual ringbuffer used by printk)::
 *
 *        DEFINE_PRINTKRB(test_rb, 15, 5);
 *
 * This ringbuffer allows up to 32768 records (2 ^ 15) and has a size of
 * 1 MiB (2 ^ (15 + 5)) for text data.
 *
 * Sample writer code::
 *
 *        const char *textstr = "message text";
 *        struct prb_reserved_entry e;
 *        struct printk_record r;
 *
 *        // specify how much to allocate
 *        prb_rec_init_wr(&r, strlen(textstr) + 1);
 *
 *        if (prb_reserve(&e, &test_rb, &r)) {
 *                snprintf(r.text_buf, r.text_buf_size, "%s", textstr);
 *
 *                r.info->text_len = strlen(textstr);
 *                r.info->ts_nsec = local_clock();
 *                r.info->caller_id = printk_caller_id();
 *
 *                // commit and finalize the record
 *                prb_final_commit(&e);
 *        }
 *
 * Note that additional writer functions are available to extend a record
 * after it has been committed but not yet finalized. This can be done as
 * long as no new records have been reserved and the caller is the same.
 *
 * Sample writer code (record extending)::
 *
 *                // alternate rest of previous example
 *
 *                r.info->text_len = strlen(textstr);
 *                r.info->ts_nsec = local_clock();
 *                r.info->caller_id = printk_caller_id();
 *
 *                // commit the record (but do not finalize yet)
 *                prb_commit(&e);
 *        }
 *
 *        ...
 *
 *        // specify additional 5 bytes text space to extend
 *        prb_rec_init_wr(&r, 5);
 *
 *        // try to extend, but only if it does not exceed 32 bytes
 *        if (prb_reserve_in_last(&e, &test_rb, &r, printk_caller_id(), 32)) {
 *                snprintf(&r.text_buf[r.info->text_len],
 *                         r.text_buf_size - r.info->text_len, "hello");
 *
 *                r.info->text_len += 5;
 *
 *                // commit and finalize the record
 *                prb_final_commit(&e);
 *        }
 *
 * Sample reader code::
 *
 *        struct printk_info info;
 *        struct printk_record r;
 *        char text_buf[32];
 *        u64 seq;
 *
 *        prb_rec_init_rd(&r, &info, &text_buf[0], sizeof(text_buf));
 *
 *        prb_for_each_record(0, &test_rb, &seq, &r) {
 *                if (info.seq != seq)
 *                        pr_warn("lost %llu records\n", info.seq - seq);
 *
 *                if (info.text_len > r.text_buf_size) {
 *                        pr_warn("record %llu text truncated\n", info.seq);
 *                        text_buf[r.text_buf_size - 1] = 0;
 *                }
 *
 *                pr_info("%llu: %llu: %s\n", info.seq, info.ts_nsec,
 *                        &text_buf[0]);
 *        }
 *
 * Note that additional less convenient reader functions are available to
 * allow complex record access.
 *
 * ABA Issues
 * ~~~~~~~~~~
 * To help avoid ABA issues, descriptors are referenced by IDs (array index
 * values combined with tagged bits counting array wraps) and data blocks are
 * referenced by logical positions (array index values combined with tagged
 * bits counting array wraps). However, on 32-bit systems the number of
 * tagged bits is relatively small such that an ABA incident is (at least
 * theoretically) possible. For example, if 4 million maximally sized (1KiB)
 * printk messages were to occur in NMI context on a 32-bit system, the
 * interrupted context would not be able to recognize that the 32-bit integer
 * completely wrapped and thus represents a different data block than the one
 * the interrupted context expects.
 *
 * To help combat this possibility, additional state checking is performed
 * (such as using cmpxchg() even though set() would suffice). These extra
 * checks are commented as such and will hopefully catch any ABA issue that
 * a 32-bit system might experience.
 *
 * Memory Barriers
 * ~~~~~~~~~~~~~~~
 * Multiple memory barriers are used. To simplify proving correctness and
 * generating litmus tests, lines of code related to memory barriers
 * (loads, stores, and the associated memory barriers) are labeled::
 *
 *        LMM(function:letter)
 *
 * Comments reference the labels using only the "function:letter" part.
 *
 * The memory barrier pairs and their ordering are:
 *
 *   desc_reserve:D / desc_reserve:B
 *     push descriptor tail (id), then push descriptor head (id)
 *
 *   desc_reserve:D / data_push_tail:B
 *     push data tail (lpos), then set new descriptor reserved (state)
 *
 *   desc_reserve:D / desc_push_tail:C
 *     push descriptor tail (id), then set new descriptor reserved (state)
 *
 *   desc_reserve:D / prb_first_seq:C
 *     push descriptor tail (id), then set new descriptor reserved (state)
 *
 *   desc_reserve:F / desc_read:D
 *     set new descriptor id and reserved (state), then allow writer changes
 *
 *   data_alloc:A (or data_realloc:A) / desc_read:D
 *     set old descriptor reusable (state), then modify new data block area
 *
 *   data_alloc:A (or data_realloc:A) / data_push_tail:B
 *     push data tail (lpos), then modify new data block area
 *
 *   _prb_commit:B / desc_read:B
 *     store writer changes, then set new descriptor committed (state)
 *
 *   desc_reopen_last:A / _prb_commit:B
 *     set descriptor reserved (state), then read descriptor data
 *
 *   _prb_commit:B / desc_reserve:D
 *     set new descriptor committed (state), then check descriptor head (id)
 *
 *   data_push_tail:D / data_push_tail:A
 *     set descriptor reusable (state), then push data tail (lpos)
 *
 *   desc_push_tail:B / desc_reserve:D
 *     set descriptor reusable (state), then push descriptor tail (id)
 *
 *   desc_update_last_finalized:A / desc_last_finalized_seq:A
 *     store finalized record, then set new highest finalized sequence number
 */

#define DATA_SIZE(data_ring)                _DATA_SIZE((data_ring)->size_bits)
#define DATA_SIZE_MASK(data_ring)        (DATA_SIZE(data_ring) - 1)

#define DESCS_COUNT(desc_ring)                _DESCS_COUNT((desc_ring)->count_bits)
#define DESCS_COUNT_MASK(desc_ring)        (DESCS_COUNT(desc_ring) - 1)

/* Determine the data array index from a logical position. */
#define DATA_INDEX(data_ring, lpos)        ((lpos) & DATA_SIZE_MASK(data_ring))

/* Determine the desc array index from an ID or sequence number. */
#define DESC_INDEX(desc_ring, n)        ((n) & DESCS_COUNT_MASK(desc_ring))

/* Determine how many times the data array has wrapped. */
#define DATA_WRAPS(data_ring, lpos)        ((lpos) >> (data_ring)->size_bits)

/* Determine if a logical position refers to a data-less block. */
#define LPOS_DATALESS(lpos)                ((lpos) & 1UL)
#define BLK_DATALESS(blk)                (LPOS_DATALESS((blk)->begin) && \
                                         LPOS_DATALESS((blk)->next))

/* Get the logical position at index 0 of the current wrap. */
#define DATA_THIS_WRAP_START_LPOS(data_ring, lpos) \
((lpos) & ~DATA_SIZE_MASK(data_ring))

/* Get the ID for the same index of the previous wrap as the given ID. */
#define DESC_ID_PREV_WRAP(desc_ring, id) \
DESC_ID((id) - DESCS_COUNT(desc_ring))

/*
 * A data block: mapped directly to the beginning of the data block area
 * specified as a logical position within the data ring.
 *
 * @id:   the ID of the associated descriptor
 * @data: the writer data
 *
 * Note that the size of a data block is only known by its associated
 * descriptor.
 */
struct prb_data_block {
        unsigned long        id;
        char                data[];
};

/*
 * Return the descriptor associated with @n. @n can be either a
 * descriptor ID or a sequence number.
 */
static struct prb_desc *to_desc(struct prb_desc_ring *desc_ring, u64 n)
{
        return &desc_ring->descs[DESC_INDEX(desc_ring, n)];
}

/*
 * Return the printk_info associated with @n. @n can be either a
 * descriptor ID or a sequence number.
 */
static struct printk_info *to_info(struct prb_desc_ring *desc_ring, u64 n)
{
        return &desc_ring->infos[DESC_INDEX(desc_ring, n)];
}

static struct prb_data_block *to_block(struct prb_data_ring *data_ring,
                                       unsigned long begin_lpos)
{
        return (void *)&data_ring->data[DATA_INDEX(data_ring, begin_lpos)];
}

/*
 * Increase the data size to account for data block meta data plus any
 * padding so that the adjacent data block is aligned on the ID size.
 */
static unsigned int to_blk_size(unsigned int size)
{
        struct prb_data_block *db = NULL;

        size += sizeof(*db);
        size = ALIGN(size, sizeof(db->id));
        return size;
}

/*
 * Sanity checker for reserve size. The ringbuffer code assumes that a data
 * block does not exceed the maximum possible size that could fit within the
 * ringbuffer. This function provides that basic size check so that the
 * assumption is safe.
 */
static bool data_check_size(struct prb_data_ring *data_ring, unsigned int size)
{
        struct prb_data_block *db = NULL;

        if (size == 0)
                return true;

        /*
         * Ensure the alignment padded size could possibly fit in the data
         * array. The largest possible data block must still leave room for
         * at least the ID of the next block.
         */
        size = to_blk_size(size);
        if (size > DATA_SIZE(data_ring) - sizeof(db->id))
                return false;

        return true;
}

/* Query the state of a descriptor. */
static enum desc_state get_desc_state(unsigned long id,
                                      unsigned long state_val)
{
        if (id != DESC_ID(state_val))
                return desc_miss;

        return DESC_STATE(state_val);
}

/*
 * Get a copy of a specified descriptor and return its queried state. If the
 * descriptor is in an inconsistent state (miss or reserved), the caller can
 * only expect the descriptor's @state_var field to be valid.
 *
 * The sequence number and caller_id can be optionally retrieved. Like all
 * non-state_var data, they are only valid if the descriptor is in a
 * consistent state.
 */
static enum desc_state desc_read(struct prb_desc_ring *desc_ring,
                                 unsigned long id, struct prb_desc *desc_out,
                                 u64 *seq_out, u32 *caller_id_out)
{
        struct printk_info *info = to_info(desc_ring, id);
        struct prb_desc *desc = to_desc(desc_ring, id);
        atomic_long_t *state_var = &desc->state_var;
        enum desc_state d_state;
        unsigned long state_val;

        /* Check the descriptor state. */
        state_val = atomic_long_read(state_var); /* LMM(desc_read:A) */
        d_state = get_desc_state(id, state_val);
        if (d_state == desc_miss || d_state == desc_reserved) {
                /*
                 * The descriptor is in an inconsistent state. Set at least
                 * @state_var so that the caller can see the details of
                 * the inconsistent state.
                 */
                goto out;
        }

        /*
         * Guarantee the state is loaded before copying the descriptor
         * content. This avoids copying obsolete descriptor content that might
         * not apply to the descriptor state. This pairs with _prb_commit:B.
         *
         * Memory barrier involvement:
         *
         * If desc_read:A reads from _prb_commit:B, then desc_read:C reads
         * from _prb_commit:A.
         *
         * Relies on:
         *
         * WMB from _prb_commit:A to _prb_commit:B
         *    matching
         * RMB from desc_read:A to desc_read:C
         */
        smp_rmb(); /* LMM(desc_read:B) */

        /*
         * Copy the descriptor data. The data is not valid until the
         * state has been re-checked. A memcpy() for all of @desc
         * cannot be used because of the atomic_t @state_var field.
         */
        if (desc_out) {
                memcpy(&desc_out->text_blk_lpos, &desc->text_blk_lpos,
                       sizeof(desc_out->text_blk_lpos)); /* LMM(desc_read:C) */
        }
        if (seq_out)
                *seq_out = info->seq; /* also part of desc_read:C */
        if (caller_id_out)
                *caller_id_out = info->caller_id; /* also part of desc_read:C */

        /*
         * 1. Guarantee the descriptor content is loaded before re-checking
         *    the state. This avoids reading an obsolete descriptor state
         *    that may not apply to the copied content. This pairs with
         *    desc_reserve:F.
         *
         *    Memory barrier involvement:
         *
         *    If desc_read:C reads from desc_reserve:G, then desc_read:E
         *    reads from desc_reserve:F.
         *
         *    Relies on:
         *
         *    WMB from desc_reserve:F to desc_reserve:G
         *       matching
         *    RMB from desc_read:C to desc_read:E
         *
         * 2. Guarantee the record data is loaded before re-checking the
         *    state. This avoids reading an obsolete descriptor state that may
         *    not apply to the copied data. This pairs with data_alloc:A and
         *    data_realloc:A.
         *
         *    Memory barrier involvement:
         *
         *    If copy_data:A reads from data_alloc:B, then desc_read:E
         *    reads from desc_make_reusable:A.
         *
         *    Relies on:
         *
         *    MB from desc_make_reusable:A to data_alloc:B
         *       matching
         *    RMB from desc_read:C to desc_read:E
         *
         *    Note: desc_make_reusable:A and data_alloc:B can be different
         *          CPUs. However, the data_alloc:B CPU (which performs the
         *          full memory barrier) must have previously seen
         *          desc_make_reusable:A.
         */
        smp_rmb(); /* LMM(desc_read:D) */

        /*
         * The data has been copied. Return the current descriptor state,
         * which may have changed since the load above.
         */
        state_val = atomic_long_read(state_var); /* LMM(desc_read:E) */
        d_state = get_desc_state(id, state_val);
out:
        if (desc_out)
                atomic_long_set(&desc_out->state_var, state_val);
        return d_state;
}

/*
 * Take a specified descriptor out of the finalized state by attempting
 * the transition from finalized to reusable. Either this context or some
 * other context will have been successful.
 */
static void desc_make_reusable(struct prb_desc_ring *desc_ring,
                               unsigned long id)
{
        unsigned long val_finalized = DESC_SV(id, desc_finalized);
        unsigned long val_reusable = DESC_SV(id, desc_reusable);
        struct prb_desc *desc = to_desc(desc_ring, id);
        atomic_long_t *state_var = &desc->state_var;

        atomic_long_cmpxchg_relaxed(state_var, val_finalized,
                                    val_reusable); /* LMM(desc_make_reusable:A) */
}

/*
 * Given the text data ring, put the associated descriptor of each
 * data block from @lpos_begin until @lpos_end into the reusable state.
 *
 * If there is any problem making the associated descriptor reusable, either
 * the descriptor has not yet been finalized or another writer context has
 * already pushed the tail lpos past the problematic data block. Regardless,
 * on error the caller can re-load the tail lpos to determine the situation.
 */
static bool data_make_reusable(struct printk_ringbuffer *rb,
                               unsigned long lpos_begin,
                               unsigned long lpos_end,
                               unsigned long *lpos_out)
{

        struct prb_data_ring *data_ring = &rb->text_data_ring;
        struct prb_desc_ring *desc_ring = &rb->desc_ring;
        struct prb_data_block *blk;
        enum desc_state d_state;
        struct prb_desc desc;
        struct prb_data_blk_lpos *blk_lpos = &desc.text_blk_lpos;
        unsigned long id;

        /* Loop until @lpos_begin has advanced to or beyond @lpos_end. */
        while ((lpos_end - lpos_begin) - 1 < DATA_SIZE(data_ring)) {
                blk = to_block(data_ring, lpos_begin);

                /*
                 * Load the block ID from the data block. This is a data race
                 * against a writer that may have newly reserved this data
                 * area. If the loaded value matches a valid descriptor ID,
                 * the blk_lpos of that descriptor will be checked to make
                 * sure it points back to this data block. If the check fails,
                 * the data area has been recycled by another writer.
                 */
                id = blk->id; /* LMM(data_make_reusable:A) */

                d_state = desc_read(desc_ring, id, &desc,
                                    NULL, NULL); /* LMM(data_make_reusable:B) */

                switch (d_state) {
                case desc_miss:
                case desc_reserved:
                case desc_committed:
                        return false;
                case desc_finalized:
                        /*
                         * This data block is invalid if the descriptor
                         * does not point back to it.
                         */
                        if (blk_lpos->begin != lpos_begin)
                                return false;
                        desc_make_reusable(desc_ring, id);
                        break;
                case desc_reusable:
                        /*
                         * This data block is invalid if the descriptor
                         * does not point back to it.
                         */
                        if (blk_lpos->begin != lpos_begin)
                                return false;
                        break;
                }

                /* Advance @lpos_begin to the next data block. */
                lpos_begin = blk_lpos->next;
        }

        *lpos_out = lpos_begin;
        return true;
}

/*
 * Advance the data ring tail to at least @lpos. This function puts
 * descriptors into the reusable state if the tail is pushed beyond
 * their associated data block.
 */
static bool data_push_tail(struct printk_ringbuffer *rb, unsigned long lpos)
{
        struct prb_data_ring *data_ring = &rb->text_data_ring;
        unsigned long tail_lpos_new;
        unsigned long tail_lpos;
        unsigned long next_lpos;

        /* If @lpos is from a data-less block, there is nothing to do. */
        if (LPOS_DATALESS(lpos))
                return true;

        /*
         * Any descriptor states that have transitioned to reusable due to the
         * data tail being pushed to this loaded value will be visible to this
         * CPU. This pairs with data_push_tail:D.
         *
         * Memory barrier involvement:
         *
         * If data_push_tail:A reads from data_push_tail:D, then this CPU can
         * see desc_make_reusable:A.
         *
         * Relies on:
         *
         * MB from desc_make_reusable:A to data_push_tail:D
         *    matches
         * READFROM from data_push_tail:D to data_push_tail:A
         *    thus
         * READFROM from desc_make_reusable:A to this CPU
         */
        tail_lpos = atomic_long_read(&data_ring->tail_lpos); /* LMM(data_push_tail:A) */

        /*
         * Loop until the tail lpos is at or beyond @lpos. This condition
         * may already be satisfied, resulting in no full memory barrier
         * from data_push_tail:D being performed. However, since this CPU
         * sees the new tail lpos, any descriptor states that transitioned to
         * the reusable state must already be visible.
         */
        while ((lpos - tail_lpos) - 1 < DATA_SIZE(data_ring)) {
                /*
                 * Make all descriptors reusable that are associated with
                 * data blocks before @lpos.
                 */
                if (!data_make_reusable(rb, tail_lpos, lpos, &next_lpos)) {
                        /*
                         * 1. Guarantee the block ID loaded in
                         *    data_make_reusable() is performed before
                         *    reloading the tail lpos. The failed
                         *    data_make_reusable() may be due to a newly
                         *    recycled data area causing the tail lpos to
                         *    have been previously pushed. This pairs with
                         *    data_alloc:A and data_realloc:A.
                         *
                         *    Memory barrier involvement:
                         *
                         *    If data_make_reusable:A reads from data_alloc:B,
                         *    then data_push_tail:C reads from
                         *    data_push_tail:D.
                         *
                         *    Relies on:
                         *
                         *    MB from data_push_tail:D to data_alloc:B
                         *       matching
                         *    RMB from data_make_reusable:A to
                         *    data_push_tail:C
                         *
                         *    Note: data_push_tail:D and data_alloc:B can be
                         *          different CPUs. However, the data_alloc:B
                         *          CPU (which performs the full memory
                         *          barrier) must have previously seen
                         *          data_push_tail:D.
                         *
                         * 2. Guarantee the descriptor state loaded in
                         *    data_make_reusable() is performed before
                         *    reloading the tail lpos. The failed
                         *    data_make_reusable() may be due to a newly
                         *    recycled descriptor causing the tail lpos to
                         *    have been previously pushed. This pairs with
                         *    desc_reserve:D.
                         *
                         *    Memory barrier involvement:
                         *
                         *    If data_make_reusable:B reads from
                         *    desc_reserve:F, then data_push_tail:C reads
                         *    from data_push_tail:D.
                         *
                         *    Relies on:
                         *
                         *    MB from data_push_tail:D to desc_reserve:F
                         *       matching
                         *    RMB from data_make_reusable:B to
                         *    data_push_tail:C
                         *
                         *    Note: data_push_tail:D and desc_reserve:F can
                         *          be different CPUs. However, the
                         *          desc_reserve:F CPU (which performs the
                         *          full memory barrier) must have previously
                         *          seen data_push_tail:D.
                         */
                        smp_rmb(); /* LMM(data_push_tail:B) */

                        tail_lpos_new = atomic_long_read(&data_ring->tail_lpos
                                                        ); /* LMM(data_push_tail:C) */
                        if (tail_lpos_new == tail_lpos)
                                return false;

                        /* Another CPU pushed the tail. Try again. */
                        tail_lpos = tail_lpos_new;
                        continue;
                }

                /*
                 * Guarantee any descriptor states that have transitioned to
                 * reusable are stored before pushing the tail lpos. A full
                 * memory barrier is needed since other CPUs may have made
                 * the descriptor states reusable. This pairs with
                 * data_push_tail:A.
                 */
                if (atomic_long_try_cmpxchg(&data_ring->tail_lpos, &tail_lpos,
                                            next_lpos)) { /* LMM(data_push_tail:D) */
                        break;
                }
        }

        return true;
}

/*
 * Advance the desc ring tail. This function advances the tail by one
 * descriptor, thus invalidating the oldest descriptor. Before advancing
 * the tail, the tail descriptor is made reusable and all data blocks up to
 * and including the descriptor's data block are invalidated (i.e. the data
 * ring tail is pushed past the data block of the descriptor being made
 * reusable).
 */
static bool desc_push_tail(struct printk_ringbuffer *rb,
                           unsigned long tail_id)
{
        struct prb_desc_ring *desc_ring = &rb->desc_ring;
        enum desc_state d_state;
        struct prb_desc desc;

        d_state = desc_read(desc_ring, tail_id, &desc, NULL, NULL);

        switch (d_state) {
        case desc_miss:
                /*
                 * If the ID is exactly 1 wrap behind the expected, it is
                 * in the process of being reserved by another writer and
                 * must be considered reserved.
                 */
                if (DESC_ID(atomic_long_read(&desc.state_var)) ==
                    DESC_ID_PREV_WRAP(desc_ring, tail_id)) {
                        return false;
                }

                /*
                 * The ID has changed. Another writer must have pushed the
                 * tail and recycled the descriptor already. Success is
                 * returned because the caller is only interested in the
                 * specified tail being pushed, which it was.
                 */
                return true;
        case desc_reserved:
        case desc_committed:
                return false;
        case desc_finalized:
                desc_make_reusable(desc_ring, tail_id);
                break;
        case desc_reusable:
                break;
        }

        /*
         * Data blocks must be invalidated before their associated
         * descriptor can be made available for recycling. Invalidating
         * them later is not possible because there is no way to trust
         * data blocks once their associated descriptor is gone.
         */

        if (!data_push_tail(rb, desc.text_blk_lpos.next))
                return false;

        /*
         * Check the next descriptor after @tail_id before pushing the tail
         * to it because the tail must always be in a finalized or reusable
         * state. The implementation of prb_first_seq() relies on this.
         *
         * A successful read implies that the next descriptor is less than or
         * equal to @head_id so there is no risk of pushing the tail past the
         * head.
         */
        d_state = desc_read(desc_ring, DESC_ID(tail_id + 1), &desc,
                            NULL, NULL); /* LMM(desc_push_tail:A) */

        if (d_state == desc_finalized || d_state == desc_reusable) {
                /*
                 * Guarantee any descriptor states that have transitioned to
                 * reusable are stored before pushing the tail ID. This allows
                 * verifying the recycled descriptor state. A full memory
                 * barrier is needed since other CPUs may have made the
                 * descriptor states reusable. This pairs with desc_reserve:D.
                 */
                atomic_long_cmpxchg(&desc_ring->tail_id, tail_id,
                                    DESC_ID(tail_id + 1)); /* LMM(desc_push_tail:B) */
        } else {
                /*
                 * Guarantee the last state load from desc_read() is before
                 * reloading @tail_id in order to see a new tail ID in the
                 * case that the descriptor has been recycled. This pairs
                 * with desc_reserve:D.
                 *
                 * Memory barrier involvement:
                 *
                 * If desc_push_tail:A reads from desc_reserve:F, then
                 * desc_push_tail:D reads from desc_push_tail:B.
                 *
                 * Relies on:
                 *
                 * MB from desc_push_tail:B to desc_reserve:F
                 *    matching
                 * RMB from desc_push_tail:A to desc_push_tail:D
                 *
                 * Note: desc_push_tail:B and desc_reserve:F can be different
                 *       CPUs. However, the desc_reserve:F CPU (which performs
                 *       the full memory barrier) must have previously seen
                 *       desc_push_tail:B.
                 */
                smp_rmb(); /* LMM(desc_push_tail:C) */

                /*
                 * Re-check the tail ID. The descriptor following @tail_id is
                 * not in an allowed tail state. But if the tail has since
                 * been moved by another CPU, then it does not matter.
                 */
                if (atomic_long_read(&desc_ring->tail_id) == tail_id) /* LMM(desc_push_tail:D) */
                        return false;
        }

        return true;
}

/* Reserve a new descriptor, invalidating the oldest if necessary. */
static bool desc_reserve(struct printk_ringbuffer *rb, unsigned long *id_out)
{
        struct prb_desc_ring *desc_ring = &rb->desc_ring;
        unsigned long prev_state_val;
        unsigned long id_prev_wrap;
        struct prb_desc *desc;
        unsigned long head_id;
        unsigned long id;

        head_id = atomic_long_read(&desc_ring->head_id); /* LMM(desc_reserve:A) */

        do {
                id = DESC_ID(head_id + 1);
                id_prev_wrap = DESC_ID_PREV_WRAP(desc_ring, id);

                /*
                 * Guarantee the head ID is read before reading the tail ID.
                 * Since the tail ID is updated before the head ID, this
                 * guarantees that @id_prev_wrap is never ahead of the tail
                 * ID. This pairs with desc_reserve:D.
                 *
                 * Memory barrier involvement:
                 *
                 * If desc_reserve:A reads from desc_reserve:D, then
                 * desc_reserve:C reads from desc_push_tail:B.
                 *
                 * Relies on:
                 *
                 * MB from desc_push_tail:B to desc_reserve:D
                 *    matching
                 * RMB from desc_reserve:A to desc_reserve:C
                 *
                 * Note: desc_push_tail:B and desc_reserve:D can be different
                 *       CPUs. However, the desc_reserve:D CPU (which performs
                 *       the full memory barrier) must have previously seen
                 *       desc_push_tail:B.
                 */
                smp_rmb(); /* LMM(desc_reserve:B) */

                if (id_prev_wrap == atomic_long_read(&desc_ring->tail_id
                                                    )) { /* LMM(desc_reserve:C) */
                        /*
                         * Make space for the new descriptor by
                         * advancing the tail.
                         */
                        if (!desc_push_tail(rb, id_prev_wrap))
                                return false;
                }

                /*
                 * 1. Guarantee the tail ID is read before validating the
                 *    recycled descriptor state. A read memory barrier is
                 *    sufficient for this. This pairs with desc_push_tail:B.
                 *
                 *    Memory barrier involvement:
                 *
                 *    If desc_reserve:C reads from desc_push_tail:B, then
                 *    desc_reserve:E reads from desc_make_reusable:A.
                 *
                 *    Relies on:
                 *
                 *    MB from desc_make_reusable:A to desc_push_tail:B
                 *       matching
                 *    RMB from desc_reserve:C to desc_reserve:E
                 *
                 *    Note: desc_make_reusable:A and desc_push_tail:B can be
                 *          different CPUs. However, the desc_push_tail:B CPU
                 *          (which performs the full memory barrier) must have
                 *          previously seen desc_make_reusable:A.
                 *
                 * 2. Guarantee the tail ID is stored before storing the head
                 *    ID. This pairs with desc_reserve:B.
                 *
                 * 3. Guarantee any data ring tail changes are stored before
                 *    recycling the descriptor. Data ring tail changes can
                 *    happen via desc_push_tail()->data_push_tail(). A full
                 *    memory barrier is needed since another CPU may have
                 *    pushed the data ring tails. This pairs with
                 *    data_push_tail:B.
                 *
                 * 4. Guarantee a new tail ID is stored before recycling the
                 *    descriptor. A full memory barrier is needed since
                 *    another CPU may have pushed the tail ID. This pairs
                 *    with desc_push_tail:C and this also pairs with
                 *    prb_first_seq:C.
                 *
                 * 5. Guarantee the head ID is stored before trying to
                 *    finalize the previous descriptor. This pairs with
                 *    _prb_commit:B.
                 */
        } while (!atomic_long_try_cmpxchg(&desc_ring->head_id, &head_id,
                                          id)); /* LMM(desc_reserve:D) */

        desc = to_desc(desc_ring, id);

        /*
         * If the descriptor has been recycled, verify the old state val.
         * See "ABA Issues" about why this verification is performed.
         */
        prev_state_val = atomic_long_read(&desc->state_var); /* LMM(desc_reserve:E) */
        if (prev_state_val &&
            get_desc_state(id_prev_wrap, prev_state_val) != desc_reusable) {
                WARN_ON_ONCE(1);
                return false;
        }

        /*
         * Assign the descriptor a new ID and set its state to reserved.
         * See "ABA Issues" about why cmpxchg() instead of set() is used.
         *
         * Guarantee the new descriptor ID and state is stored before making
         * any other changes. A write memory barrier is sufficient for this.
         * This pairs with desc_read:D.
         */
        if (!atomic_long_try_cmpxchg(&desc->state_var, &prev_state_val,
                        DESC_SV(id, desc_reserved))) { /* LMM(desc_reserve:F) */
                WARN_ON_ONCE(1);
                return false;
        }

        /* Now data in @desc can be modified: LMM(desc_reserve:G) */

        *id_out = id;
        return true;
}

/* Determine the end of a data block. */
static unsigned long get_next_lpos(struct prb_data_ring *data_ring,
                                   unsigned long lpos, unsigned int size)
{
        unsigned long begin_lpos;
        unsigned long next_lpos;

        begin_lpos = lpos;
        next_lpos = lpos + size;

        /* First check if the data block does not wrap. */
        if (DATA_WRAPS(data_ring, begin_lpos) == DATA_WRAPS(data_ring, next_lpos))
                return next_lpos;

        /* Wrapping data blocks store their data at the beginning. */
        return (DATA_THIS_WRAP_START_LPOS(data_ring, next_lpos) + size);
}

/*
 * Allocate a new data block, invalidating the oldest data block(s)
 * if necessary. This function also associates the data block with
 * a specified descriptor.
 */
static char *data_alloc(struct printk_ringbuffer *rb, unsigned int size,
                        struct prb_data_blk_lpos *blk_lpos, unsigned long id)
{
        struct prb_data_ring *data_ring = &rb->text_data_ring;
        struct prb_data_block *blk;
        unsigned long begin_lpos;
        unsigned long next_lpos;

        if (size == 0) {
                /*
                 * Data blocks are not created for empty lines. Instead, the
                 * reader will recognize these special lpos values and handle
                 * it appropriately.
                 */
                blk_lpos->begin = EMPTY_LINE_LPOS;
                blk_lpos->next = EMPTY_LINE_LPOS;
                return NULL;
        }

        size = to_blk_size(size);

        begin_lpos = atomic_long_read(&data_ring->head_lpos);

        do {
                next_lpos = get_next_lpos(data_ring, begin_lpos, size);

                if (!data_push_tail(rb, next_lpos - DATA_SIZE(data_ring))) {
                        /* Failed to allocate, specify a data-less block. */
                        blk_lpos->begin = FAILED_LPOS;
                        blk_lpos->next = FAILED_LPOS;
                        return NULL;
                }

                /*
                 * 1. Guarantee any descriptor states that have transitioned
                 *    to reusable are stored before modifying the newly
                 *    allocated data area. A full memory barrier is needed
                 *    since other CPUs may have made the descriptor states
                 *    reusable. See data_push_tail:A about why the reusable
                 *    states are visible. This pairs with desc_read:D.
                 *
                 * 2. Guarantee any updated tail lpos is stored before
                 *    modifying the newly allocated data area. Another CPU may
                 *    be in data_make_reusable() and is reading a block ID
                 *    from this area. data_make_reusable() can handle reading
                 *    a garbage block ID value, but then it must be able to
                 *    load a new tail lpos. A full memory barrier is needed
                 *    since other CPUs may have updated the tail lpos. This
                 *    pairs with data_push_tail:B.
                 */
        } while (!atomic_long_try_cmpxchg(&data_ring->head_lpos, &begin_lpos,
                                          next_lpos)); /* LMM(data_alloc:A) */

        blk = to_block(data_ring, begin_lpos);
        blk->id = id; /* LMM(data_alloc:B) */

        if (DATA_WRAPS(data_ring, begin_lpos) != DATA_WRAPS(data_ring, next_lpos)) {
                /* Wrapping data blocks store their data at the beginning. */
                blk = to_block(data_ring, 0);

                /*
                 * Store the ID on the wrapped block for consistency.
                 * The printk_ringbuffer does not actually use it.
                 */
                blk->id = id;
        }

        blk_lpos->begin = begin_lpos;
        blk_lpos->next = next_lpos;

        return &blk->data[0];
}

/*
 * Try to resize an existing data block associated with the descriptor
 * specified by @id. If the resized data block should become wrapped, it
 * copies the old data to the new data block. If @size yields a data block
 * with the same or less size, the data block is left as is.
 *
 * Fail if this is not the last allocated data block or if there is not
 * enough space or it is not possible make enough space.
 *
 * Return a pointer to the beginning of the entire data buffer or NULL on
 * failure.
 */
static char *data_realloc(struct printk_ringbuffer *rb, unsigned int size,
                          struct prb_data_blk_lpos *blk_lpos, unsigned long id)
{
        struct prb_data_ring *data_ring = &rb->text_data_ring;
        struct prb_data_block *blk;
        unsigned long head_lpos;
        unsigned long next_lpos;
        bool wrapped;

        /* Reallocation only works if @blk_lpos is the newest data block. */
        head_lpos = atomic_long_read(&data_ring->head_lpos);
        if (head_lpos != blk_lpos->next)
                return NULL;

        /* Keep track if @blk_lpos was a wrapping data block. */
        wrapped = (DATA_WRAPS(data_ring, blk_lpos->begin) != DATA_WRAPS(data_ring, blk_lpos->next));

        size = to_blk_size(size);

        next_lpos = get_next_lpos(data_ring, blk_lpos->begin, size);

        /* If the data block does not increase, there is nothing to do. */
        if (head_lpos - next_lpos < DATA_SIZE(data_ring)) {
                if (wrapped)
                        blk = to_block(data_ring, 0);
                else
                        blk = to_block(data_ring, blk_lpos->begin);
                return &blk->data[0];
        }

        if (!data_push_tail(rb, next_lpos - DATA_SIZE(data_ring)))
                return NULL;

        /* The memory barrier involvement is the same as data_alloc:A. */
        if (!atomic_long_try_cmpxchg(&data_ring->head_lpos, &head_lpos,
                                     next_lpos)) { /* LMM(data_realloc:A) */
                return NULL;
        }

        blk = to_block(data_ring, blk_lpos->begin);

        if (DATA_WRAPS(data_ring, blk_lpos->begin) != DATA_WRAPS(data_ring, next_lpos)) {
                struct prb_data_block *old_blk = blk;

                /* Wrapping data blocks store their data at the beginning. */
                blk = to_block(data_ring, 0);

                /*
                 * Store the ID on the wrapped block for consistency.
                 * The printk_ringbuffer does not actually use it.
                 */
                blk->id = id;

                if (!wrapped) {
                        /*
                         * Since the allocated space is now in the newly
                         * created wrapping data block, copy the content
                         * from the old data block.
                         */
                        memcpy(&blk->data[0], &old_blk->data[0],
                               (blk_lpos->next - blk_lpos->begin) - sizeof(blk->id));
                }
        }

        blk_lpos->next = next_lpos;

        return &blk->data[0];
}

/* Return the number of bytes used by a data block. */
static unsigned int space_used(struct prb_data_ring *data_ring,
                               struct prb_data_blk_lpos *blk_lpos)
{
        /* Data-less blocks take no space. */
        if (BLK_DATALESS(blk_lpos))
                return 0;

        if (DATA_WRAPS(data_ring, blk_lpos->begin) == DATA_WRAPS(data_ring, blk_lpos->next)) {
                /* Data block does not wrap. */
                return (DATA_INDEX(data_ring, blk_lpos->next) -
                        DATA_INDEX(data_ring, blk_lpos->begin));
        }

        /*
         * For wrapping data blocks, the trailing (wasted) space is
         * also counted.
         */
        return (DATA_INDEX(data_ring, blk_lpos->next) +
                DATA_SIZE(data_ring) - DATA_INDEX(data_ring, blk_lpos->begin));
}

/*
 * Given @blk_lpos, return a pointer to the writer data from the data block
 * and calculate the size of the data part. A NULL pointer is returned if
 * @blk_lpos specifies values that could never be legal.
 *
 * This function (used by readers) performs strict validation on the lpos
 * values to possibly detect bugs in the writer code. A WARN_ON_ONCE() is
 * triggered if an internal error is detected.
 */
static const char *get_data(struct prb_data_ring *data_ring,
                            struct prb_data_blk_lpos *blk_lpos,
                            unsigned int *data_size)
{
        struct prb_data_block *db;

        /* Data-less data block description. */
        if (BLK_DATALESS(blk_lpos)) {
                /*
                 * Records that are just empty lines are also valid, even
                 * though they do not have a data block. For such records
                 * explicitly return empty string data to signify success.
                 */
                if (blk_lpos->begin == EMPTY_LINE_LPOS &&
                    blk_lpos->next == EMPTY_LINE_LPOS) {
                        *data_size = 0;
                        return "";
                }

                /* Data lost, invalid, or otherwise unavailable. */
                return NULL;
        }

        /* Regular data block: @begin less than @next and in same wrap. */
        if (DATA_WRAPS(data_ring, blk_lpos->begin) == DATA_WRAPS(data_ring, blk_lpos->next) &&
            blk_lpos->begin < blk_lpos->next) {
                db = to_block(data_ring, blk_lpos->begin);
                *data_size = blk_lpos->next - blk_lpos->begin;

        /* Wrapping data block: @begin is one wrap behind @next. */
        } else if (DATA_WRAPS(data_ring, blk_lpos->begin + DATA_SIZE(data_ring)) ==
                   DATA_WRAPS(data_ring, blk_lpos->next)) {
                db = to_block(data_ring, 0);
                *data_size = DATA_INDEX(data_ring, blk_lpos->next);

        /* Illegal block description. */
        } else {
                WARN_ON_ONCE(1);
                return NULL;
        }

        /* A valid data block will always be aligned to the ID size. */
        if (WARN_ON_ONCE(blk_lpos->begin != ALIGN(blk_lpos->begin, sizeof(db->id))) ||
            WARN_ON_ONCE(blk_lpos->next != ALIGN(blk_lpos->next, sizeof(db->id)))) {
                return NULL;
        }

        /* A valid data block will always have at least an ID. */
        if (WARN_ON_ONCE(*data_size < sizeof(db->id)))
                return NULL;

        /* Subtract block ID space from size to reflect data size. */
        *data_size -= sizeof(db->id);

        return &db->data[0];
}

/*
 * Attempt to transition the newest descriptor from committed back to reserved
 * so that the record can be modified by a writer again. This is only possible
 * if the descriptor is not yet finalized and the provided @caller_id matches.
 */
static struct prb_desc *desc_reopen_last(struct prb_desc_ring *desc_ring,
                                         u32 caller_id, unsigned long *id_out)
{
        unsigned long prev_state_val;
        enum desc_state d_state;
        struct prb_desc desc;
        struct prb_desc *d;
        unsigned long id;
        u32 cid;

        id = atomic_long_read(&desc_ring->head_id);

        /*
         * To reduce unnecessarily reopening, first check if the descriptor
         * state and caller ID are correct.
         */
        d_state = desc_read(desc_ring, id, &desc, NULL, &cid);
        if (d_state != desc_committed || cid != caller_id)
                return NULL;

        d = to_desc(desc_ring, id);

        prev_state_val = DESC_SV(id, desc_committed);

        /*
         * Guarantee the reserved state is stored before reading any
         * record data. A full memory barrier is needed because @state_var
         * modification is followed by reading. This pairs with _prb_commit:B.
         *
         * Memory barrier involvement:
         *
         * If desc_reopen_last:A reads from _prb_commit:B, then
         * prb_reserve_in_last:A reads from _prb_commit:A.
         *
         * Relies on:
         *
         * WMB from _prb_commit:A to _prb_commit:B
         *    matching
         * MB If desc_reopen_last:A to prb_reserve_in_last:A
         */
        if (!atomic_long_try_cmpxchg(&d->state_var, &prev_state_val,
                        DESC_SV(id, desc_reserved))) { /* LMM(desc_reopen_last:A) */
                return NULL;
        }

        *id_out = id;
        return d;
}

/**
 * prb_reserve_in_last() - Re-reserve and extend the space in the ringbuffer
 *                         used by the newest record.
 *
 * @e:         The entry structure to setup.
 * @rb:        The ringbuffer to re-reserve and extend data in.
 * @r:         The record structure to allocate buffers for.
 * @caller_id: The caller ID of the caller (reserving writer).
 * @max_size:  Fail if the extended size would be greater than this.
 *
 * This is the public function available to writers to re-reserve and extend
 * data.
 *
 * The writer specifies the text size to extend (not the new total size) by
 * setting the @text_buf_size field of @r. To ensure proper initialization
 * of @r, prb_rec_init_wr() should be used.
 *
 * This function will fail if @caller_id does not match the caller ID of the
 * newest record. In that case the caller must reserve new data using
 * prb_reserve().
 *
 * Context: Any context. Disables local interrupts on success.
 * Return: true if text data could be extended, otherwise false.
 *
 * On success:
 *
 *   - @r->text_buf points to the beginning of the entire text buffer.
 *
 *   - @r->text_buf_size is set to the new total size of the buffer.
 *
 *   - @r->info is not touched so that @r->info->text_len could be used
 *     to append the text.
 *
 *   - prb_record_text_space() can be used on @e to query the new
 *     actually used space.
 *
 * Important: All @r->info fields will already be set with the current values
 *            for the record. I.e. @r->info->text_len will be less than
 *            @text_buf_size. Writers can use @r->info->text_len to know
 *            where concatenation begins and writers should update
 *            @r->info->text_len after concatenating.
 */
bool prb_reserve_in_last(struct prb_reserved_entry *e, struct printk_ringbuffer *rb,
                         struct printk_record *r, u32 caller_id, unsigned int max_size)
{
        struct prb_desc_ring *desc_ring = &rb->desc_ring;
        struct printk_info *info;
        unsigned int data_size;
        struct prb_desc *d;
        unsigned long id;

        local_irq_save(e->irqflags);

        /* Transition the newest descriptor back to the reserved state. */
        d = desc_reopen_last(desc_ring, caller_id, &id);
        if (!d) {
                local_irq_restore(e->irqflags);
                goto fail_reopen;
        }

        /* Now the writer has exclusive access: LMM(prb_reserve_in_last:A) */

        info = to_info(desc_ring, id);

        /*
         * Set the @e fields here so that prb_commit() can be used if
         * anything fails from now on.
         */
        e->rb = rb;
        e->id = id;

        /*
         * desc_reopen_last() checked the caller_id, but there was no
         * exclusive access at that point. The descriptor may have
         * changed since then.
         */
        if (caller_id != info->caller_id)
                goto fail;

        if (BLK_DATALESS(&d->text_blk_lpos)) {
                if (WARN_ON_ONCE(info->text_len != 0)) {
                        pr_warn_once("wrong text_len value (%hu, expecting 0)\n",
                                     info->text_len);
                        info->text_len = 0;
                }

                if (!data_check_size(&rb->text_data_ring, r->text_buf_size))
                        goto fail;

                if (r->text_buf_size > max_size)
                        goto fail;

                r->text_buf = data_alloc(rb, r->text_buf_size,
                                         &d->text_blk_lpos, id);
        } else {
                if (!get_data(&rb->text_data_ring, &d->text_blk_lpos, &data_size))
                        goto fail;

                /*
                 * Increase the buffer size to include the original size. If
                 * the meta data (@text_len) is not sane, use the full data
                 * block size.
                 */
                if (WARN_ON_ONCE(info->text_len > data_size)) {
                        pr_warn_once("wrong text_len value (%hu, expecting <=%u)\n",
                                     info->text_len, data_size);
                        info->text_len = data_size;
                }
                r->text_buf_size += info->text_len;

                if (!data_check_size(&rb->text_data_ring, r->text_buf_size))
                        goto fail;

                if (r->text_buf_size > max_size)
                        goto fail;

                r->text_buf = data_realloc(rb, r->text_buf_size,
                                           &d->text_blk_lpos, id);
        }
        if (r->text_buf_size && !r->text_buf)
                goto fail;

        r->info = info;

        e->text_space = space_used(&rb->text_data_ring, &d->text_blk_lpos);

        return true;
fail:
        prb_commit(e);
        /* prb_commit() re-enabled interrupts. */
fail_reopen:
        /* Make it clear to the caller that the re-reserve failed. */
        memset(r, 0, sizeof(*r));
        return false;
}

/*
 * @last_finalized_seq value guarantees that all records up to and including
 * this sequence number are finalized and can be read. The only exception are
 * too old records which have already been overwritten.
 *
 * It is also guaranteed that @last_finalized_seq only increases.
 *
 * Be aware that finalized records following non-finalized records are not
 * reported because they are not yet available to the reader. For example,
 * a new record stored via printk() will not be available to a printer if
 * it follows a record that has not been finalized yet. However, once that
 * non-finalized record becomes finalized, @last_finalized_seq will be
 * appropriately updated and the full set of finalized records will be
 * available to the printer. And since each printk() caller will either
 * directly print or trigger deferred printing of all available unprinted
 * records, all printk() messages will get printed.
 */
static u64 desc_last_finalized_seq(struct printk_ringbuffer *rb)
{
        struct prb_desc_ring *desc_ring = &rb->desc_ring;
        unsigned long ulseq;

        /*
         * Guarantee the sequence number is loaded before loading the
         * associated record in order to guarantee that the record can be
         * seen by this CPU. This pairs with desc_update_last_finalized:A.
         */
        ulseq = atomic_long_read_acquire(&desc_ring->last_finalized_seq
                                        ); /* LMM(desc_last_finalized_seq:A) */

        return __ulseq_to_u64seq(rb, ulseq);
}

static bool _prb_read_valid(struct printk_ringbuffer *rb, u64 *seq,
                            struct printk_record *r, unsigned int *line_count);

/*
 * Check if there are records directly following @last_finalized_seq that are
 * finalized. If so, update @last_finalized_seq to the latest of these
 * records. It is not allowed to skip over records that are not yet finalized.
 */
static void desc_update_last_finalized(struct printk_ringbuffer *rb)
{
        struct prb_desc_ring *desc_ring = &rb->desc_ring;
        u64 old_seq = desc_last_finalized_seq(rb);
        unsigned long oldval;
        unsigned long newval;
        u64 finalized_seq;
        u64 try_seq;

try_again:
        finalized_seq = old_seq;
        try_seq = finalized_seq + 1;

        /* Try to find later finalized records. */
        while (_prb_read_valid(rb, &try_seq, NULL, NULL)) {
                finalized_seq = try_seq;
                try_seq++;
        }

        /* No update needed if no later finalized record was found. */
        if (finalized_seq == old_seq)
                return;

        oldval = __u64seq_to_ulseq(old_seq);
        newval = __u64seq_to_ulseq(finalized_seq);

        /*
         * Set the sequence number of a later finalized record that has been
         * seen.
         *
         * Guarantee the record data is visible to other CPUs before storing
         * its sequence number. This pairs with desc_last_finalized_seq:A.
         *
         * Memory barrier involvement:
         *
         * If desc_last_finalized_seq:A reads from
         * desc_update_last_finalized:A, then desc_read:A reads from
         * _prb_commit:B.
         *
         * Relies on:
         *
         * RELEASE from _prb_commit:B to desc_update_last_finalized:A
         *    matching
         * ACQUIRE from desc_last_finalized_seq:A to desc_read:A
         *
         * Note: _prb_commit:B and desc_update_last_finalized:A can be
         *       different CPUs. However, the desc_update_last_finalized:A
         *       CPU (which performs the release) must have previously seen
         *       _prb_commit:B.
         */
        if (!atomic_long_try_cmpxchg_release(&desc_ring->last_finalized_seq,
                                &oldval, newval)) { /* LMM(desc_update_last_finalized:A) */
                old_seq = __ulseq_to_u64seq(rb, oldval);
                goto try_again;
        }
}

/*
 * Attempt to finalize a specified descriptor. If this fails, the descriptor
 * is either already final or it will finalize itself when the writer commits.
 */
static void desc_make_final(struct printk_ringbuffer *rb, unsigned long id)
{
        struct prb_desc_ring *desc_ring = &rb->desc_ring;
        unsigned long prev_state_val = DESC_SV(id, desc_committed);
        struct prb_desc *d = to_desc(desc_ring, id);

        if (atomic_long_try_cmpxchg_relaxed(&d->state_var, &prev_state_val,
                        DESC_SV(id, desc_finalized))) { /* LMM(desc_make_final:A) */
                desc_update_last_finalized(rb);
        }
}

/**
 * prb_reserve() - Reserve space in the ringbuffer.
 *
 * @e:  The entry structure to setup.
 * @rb: The ringbuffer to reserve data in.
 * @r:  The record structure to allocate buffers for.
 *
 * This is the public function available to writers to reserve data.
 *
 * The writer specifies the text size to reserve by setting the
 * @text_buf_size field of @r. To ensure proper initialization of @r,
 * prb_rec_init_wr() should be used.
 *
 * Context: Any context. Disables local interrupts on success.
 * Return: true if at least text data could be allocated, otherwise false.
 *
 * On success, the fields @info and @text_buf of @r will be set by this
 * function and should be filled in by the writer before committing. Also
 * on success, prb_record_text_space() can be used on @e to query the actual
 * space used for the text data block.
 *
 * Important: @info->text_len needs to be set correctly by the writer in
 *            order for data to be readable and/or extended. Its value
 *            is initialized to 0.
 */
bool prb_reserve(struct prb_reserved_entry *e, struct printk_ringbuffer *rb,
                 struct printk_record *r)
{
        struct prb_desc_ring *desc_ring = &rb->desc_ring;
        struct printk_info *info;
        struct prb_desc *d;
        unsigned long id;
        u64 seq;

        if (!data_check_size(&rb->text_data_ring, r->text_buf_size))
                goto fail;

        /*
         * Descriptors in the reserved state act as blockers to all further
         * reservations once the desc_ring has fully wrapped. Disable
         * interrupts during the reserve/commit window in order to minimize
         * the likelihood of this happening.
         */
        local_irq_save(e->irqflags);

        if (!desc_reserve(rb, &id)) {
                /* Descriptor reservation failures are tracked. */
                atomic_long_inc(&rb->fail);
                local_irq_restore(e->irqflags);
                goto fail;
        }

        d = to_desc(desc_ring, id);
        info = to_info(desc_ring, id);

        /*
         * All @info fields (except @seq) are cleared and must be filled in
         * by the writer. Save @seq before clearing because it is used to
         * determine the new sequence number.
         */
        seq = info->seq;
        memset(info, 0, sizeof(*info));

        /*
         * Set the @e fields here so that prb_commit() can be used if
         * text data allocation fails.
         */
        e->rb = rb;
        e->id = id;

        /*
         * Initialize the sequence number if it has "never been set".
         * Otherwise just increment it by a full wrap.
         *
         * @seq is considered "never been set" if it has a value of 0,
         * _except_ for @infos[0], which was specially setup by the ringbuffer
         * initializer and therefore is always considered as set.
         *
         * See the "Bootstrap" comment block in printk_ringbuffer.h for
         * details about how the initializer bootstraps the descriptors.
         */
        if (seq == 0 && DESC_INDEX(desc_ring, id) != 0)
                info->seq = DESC_INDEX(desc_ring, id);
        else
                info->seq = seq + DESCS_COUNT(desc_ring);

        /*
         * New data is about to be reserved. Once that happens, previous
         * descriptors are no longer able to be extended. Finalize the
         * previous descriptor now so that it can be made available to
         * readers. (For seq==0 there is no previous descriptor.)
         */
        if (info->seq > 0)
                desc_make_final(rb, DESC_ID(id - 1));

        r->text_buf = data_alloc(rb, r->text_buf_size, &d->text_blk_lpos, id);
        /* If text data allocation fails, a data-less record is committed. */
        if (r->text_buf_size && !r->text_buf) {
                prb_commit(e);
                /* prb_commit() re-enabled interrupts. */
                goto fail;
        }

        r->info = info;

        /* Record full text space used by record. */
        e->text_space = space_used(&rb->text_data_ring, &d->text_blk_lpos);

        return true;
fail:
        /* Make it clear to the caller that the reserve failed. */
        memset(r, 0, sizeof(*r));
        return false;
}

/* Commit the data (possibly finalizing it) and restore interrupts. */
static void _prb_commit(struct prb_reserved_entry *e, unsigned long state_val)
{
        struct prb_desc_ring *desc_ring = &e->rb->desc_ring;
        struct prb_desc *d = to_desc(desc_ring, e->id);
        unsigned long prev_state_val = DESC_SV(e->id, desc_reserved);

        /* Now the writer has finished all writing: LMM(_prb_commit:A) */

        /*
         * Set the descriptor as committed. See "ABA Issues" about why
         * cmpxchg() instead of set() is used.
         *
         * 1  Guarantee all record data is stored before the descriptor state
         *    is stored as committed. A write memory barrier is sufficient
         *    for this. This pairs with desc_read:B and desc_reopen_last:A.
         *
         * 2. Guarantee the descriptor state is stored as committed before
         *    re-checking the head ID in order to possibly finalize this
         *    descriptor. This pairs with desc_reserve:D.
         *
         *    Memory barrier involvement:
         *
         *    If prb_commit:A reads from desc_reserve:D, then
         *    desc_make_final:A reads from _prb_commit:B.
         *
         *    Relies on:
         *
         *    MB _prb_commit:B to prb_commit:A
         *       matching
         *    MB desc_reserve:D to desc_make_final:A
         */
        if (!atomic_long_try_cmpxchg(&d->state_var, &prev_state_val,
                        DESC_SV(e->id, state_val))) { /* LMM(_prb_commit:B) */
                WARN_ON_ONCE(1);
        }

        /* Restore interrupts, the reserve/commit window is finished. */
        local_irq_restore(e->irqflags);
}

/**
 * prb_commit() - Commit (previously reserved) data to the ringbuffer.
 *
 * @e: The entry containing the reserved data information.
 *
 * This is the public function available to writers to commit data.
 *
 * Note that the data is not yet available to readers until it is finalized.
 * Finalizing happens automatically when space for the next record is
 * reserved.
 *
 * See prb_final_commit() for a version of this function that finalizes
 * immediately.
 *
 * Context: Any context. Enables local interrupts.
 */
void prb_commit(struct prb_reserved_entry *e)
{
        struct prb_desc_ring *desc_ring = &e->rb->desc_ring;
        unsigned long head_id;

        _prb_commit(e, desc_committed);

        /*
         * If this descriptor is no longer the head (i.e. a new record has
         * been allocated), extending the data for this record is no longer
         * allowed and therefore it must be finalized.
         */
        head_id = atomic_long_read(&desc_ring->head_id); /* LMM(prb_commit:A) */
        if (head_id != e->id)
                desc_make_final(e->rb, e->id);
}

/**
 * prb_final_commit() - Commit and finalize (previously reserved) data to
 *                      the ringbuffer.
 *
 * @e: The entry containing the reserved data information.
 *
 * This is the public function available to writers to commit+finalize data.
 *
 * By finalizing, the data is made immediately available to readers.
 *
 * This function should only be used if there are no intentions of extending
 * this data using prb_reserve_in_last().
 *
 * Context: Any context. Enables local interrupts.
 */
void prb_final_commit(struct prb_reserved_entry *e)
{
        _prb_commit(e, desc_finalized);

        desc_update_last_finalized(e->rb);
}

/*
 * Count the number of lines in provided text. All text has at least 1 line
 * (even if @text_size is 0). Each '\n' processed is counted as an additional
 * line.
 */
static unsigned int count_lines(const char *text, unsigned int text_size)
{
        unsigned int next_size = text_size;
        unsigned int line_count = 1;
        const char *next = text;

        while (next_size) {
                next = memchr(next, '\n', next_size);
                if (!next)
                        break;
                line_count++;
                next++;
                next_size = text_size - (next - text);
        }

        return line_count;
}

/*
 * Given @blk_lpos, copy an expected @len of data into the provided buffer.
 * If @line_count is provided, count the number of lines in the data.
 *
 * This function (used by readers) performs strict validation on the data
 * size to possibly detect bugs in the writer code. A WARN_ON_ONCE() is
 * triggered if an internal error is detected.
 */
static bool copy_data(struct prb_data_ring *data_ring,
                      struct prb_data_blk_lpos *blk_lpos, u16 len, char *buf,
                      unsigned int buf_size, unsigned int *line_count)
{
        unsigned int data_size;
        const char *data;

        /* Caller might not want any data. */
        if ((!buf || !buf_size) && !line_count)
                return true;

        data = get_data(data_ring, blk_lpos, &data_size);
        if (!data)
                return false;

        /*
         * Actual cannot be less than expected. It can be more than expected
         * because of the trailing alignment padding.
         *
         * Note that invalid @len values can occur because the caller loads
         * the value during an allowed data race.
         */
        if (data_size < (unsigned int)len)
                return false;

        /* Caller interested in the line count? */
        if (line_count)
                *line_count = count_lines(data, len);

        /* Caller interested in the data content? */
        if (!buf || !buf_size)
                return true;

        data_size = min_t(unsigned int, buf_size, len);

        memcpy(&buf[0], data, data_size); /* LMM(copy_data:A) */
        return true;
}

/*
 * This is an extended version of desc_read(). It gets a copy of a specified
 * descriptor. However, it also verifies that the record is finalized and has
 * the sequence number @seq. On success, 0 is returned.
 *
 * Error return values:
 * -EINVAL: A finalized record with sequence number @seq does not exist.
 * -ENOENT: A finalized record with sequence number @seq exists, but its data
 *          is not available. This is a valid record, so readers should
 *          continue with the next record.
 */
static int desc_read_finalized_seq(struct prb_desc_ring *desc_ring,
                                   unsigned long id, u64 seq,
                                   struct prb_desc *desc_out)
{
        struct prb_data_blk_lpos *blk_lpos = &desc_out->text_blk_lpos;
        enum desc_state d_state;
        u64 s;

        d_state = desc_read(desc_ring, id, desc_out, &s, NULL);

        /*
         * An unexpected @id (desc_miss) or @seq mismatch means the record
         * does not exist. A descriptor in the reserved or committed state
         * means the record does not yet exist for the reader.
         */
        if (d_state == desc_miss ||
            d_state == desc_reserved ||
            d_state == desc_committed ||
            s != seq) {
                return -EINVAL;
        }

        /*
         * A descriptor in the reusable state may no longer have its data
         * available; report it as existing but with lost data. Or the record
         * may actually be a record with lost data.
         */
        if (d_state == desc_reusable ||
            (blk_lpos->begin == FAILED_LPOS && blk_lpos->next == FAILED_LPOS)) {
                return -ENOENT;
        }

        return 0;
}

/*
 * Copy the ringbuffer data from the record with @seq to the provided
 * @r buffer. On success, 0 is returned.
 *
 * See desc_read_finalized_seq() for error return values.
 */
static int prb_read(struct printk_ringbuffer *rb, u64 seq,
                    struct printk_record *r, unsigned int *line_count)
{
        struct prb_desc_ring *desc_ring = &rb->desc_ring;
        struct printk_info *info = to_info(desc_ring, seq);
        struct prb_desc *rdesc = to_desc(desc_ring, seq);
        atomic_long_t *state_var = &rdesc->state_var;
        struct prb_desc desc;
        unsigned long id;
        int err;

        /* Extract the ID, used to specify the descriptor to read. */
        id = DESC_ID(atomic_long_read(state_var));

        /* Get a local copy of the correct descriptor (if available). */
        err = desc_read_finalized_seq(desc_ring, id, seq, &desc);

        /*
         * If @r is NULL, the caller is only interested in the availability
         * of the record.
         */
        if (err || !r)
                return err;

        /* If requested, copy meta data. */
        if (r->info)
                memcpy(r->info, info, sizeof(*(r->info)));

        /* Copy text data. If it fails, this is a data-less record. */
        if (!copy_data(&rb->text_data_ring, &desc.text_blk_lpos, info->text_len,
                       r->text_buf, r->text_buf_size, line_count)) {
                return -ENOENT;
        }

        /* Ensure the record is still finalized and has the same @seq. */
        return desc_read_finalized_seq(desc_ring, id, seq, &desc);
}

/* Get the sequence number of the tail descriptor. */
u64 prb_first_seq(struct printk_ringbuffer *rb)
{
        struct prb_desc_ring *desc_ring = &rb->desc_ring;
        enum desc_state d_state;
        struct prb_desc desc;
        unsigned long id;
        u64 seq;

        for (;;) {
                id = atomic_long_read(&rb->desc_ring.tail_id); /* LMM(prb_first_seq:A) */

                d_state = desc_read(desc_ring, id, &desc, &seq, NULL); /* LMM(prb_first_seq:B) */

                /*
                 * This loop will not be infinite because the tail is
                 * _always_ in the finalized or reusable state.
                 */
                if (d_state == desc_finalized || d_state == desc_reusable)
                        break;

                /*
                 * Guarantee the last state load from desc_read() is before
                 * reloading @tail_id in order to see a new tail in the case
                 * that the descriptor has been recycled. This pairs with
                 * desc_reserve:D.
                 *
                 * Memory barrier involvement:
                 *
                 * If prb_first_seq:B reads from desc_reserve:F, then
                 * prb_first_seq:A reads from desc_push_tail:B.
                 *
                 * Relies on:
                 *
                 * MB from desc_push_tail:B to desc_reserve:F
                 *    matching
                 * RMB prb_first_seq:B to prb_first_seq:A
                 */
                smp_rmb(); /* LMM(prb_first_seq:C) */
        }

        return seq;
}

/**
 * prb_next_reserve_seq() - Get the sequence number after the most recently
 *                  reserved record.
 *
 * @rb:  The ringbuffer to get the sequence number from.
 *
 * This is the public function available to readers to see what sequence
 * number will be assigned to the next reserved record.
 *
 * Note that depending on the situation, this value can be equal to or
 * higher than the sequence number returned by prb_next_seq().
 *
 * Context: Any context.
 * Return: The sequence number that will be assigned to the next record
 *         reserved.
 */
u64 prb_next_reserve_seq(struct printk_ringbuffer *rb)
{
        struct prb_desc_ring *desc_ring = &rb->desc_ring;
        unsigned long last_finalized_id;
        atomic_long_t *state_var;
        u64 last_finalized_seq;
        unsigned long head_id;
        struct prb_desc desc;
        unsigned long diff;
        struct prb_desc *d;
        int err;

        /*
         * It may not be possible to read a sequence number for @head_id.
         * So the ID of @last_finailzed_seq is used to calculate what the
         * sequence number of @head_id will be.
         */

try_again:
        last_finalized_seq = desc_last_finalized_seq(rb);

        /*
         * @head_id is loaded after @last_finalized_seq to ensure that
         * it points to the record with @last_finalized_seq or newer.
         *
         * Memory barrier involvement:
         *
         * If desc_last_finalized_seq:A reads from
         * desc_update_last_finalized:A, then
         * prb_next_reserve_seq:A reads from desc_reserve:D.
         *
         * Relies on:
         *
         * RELEASE from desc_reserve:D to desc_update_last_finalized:A
         *    matching
         * ACQUIRE from desc_last_finalized_seq:A to prb_next_reserve_seq:A
         *
         * Note: desc_reserve:D and desc_update_last_finalized:A can be
         *       different CPUs. However, the desc_update_last_finalized:A CPU
         *       (which performs the release) must have previously seen
         *       desc_read:C, which implies desc_reserve:D can be seen.
         */
        head_id = atomic_long_read(&desc_ring->head_id); /* LMM(prb_next_reserve_seq:A) */

        d = to_desc(desc_ring, last_finalized_seq);
        state_var = &d->state_var;

        /* Extract the ID, used to specify the descriptor to read. */
        last_finalized_id = DESC_ID(atomic_long_read(state_var));

        /* Ensure @last_finalized_id is correct. */
        err = desc_read_finalized_seq(desc_ring, last_finalized_id, last_finalized_seq, &desc);

        if (err == -EINVAL) {
                if (last_finalized_seq == 0) {
                        /*
                         * No record has been finalized or even reserved yet.
                         *
                         * The @head_id is initialized such that the first
                         * increment will yield the first record (seq=0).
                         * Handle it separately to avoid a negative @diff
                         * below.
                         */
                        if (head_id == DESC0_ID(desc_ring->count_bits))
                                return 0;

                        /*
                         * One or more descriptors are already reserved. Use
                         * the descriptor ID of the first one (@seq=0) for
                         * the @diff below.
                         */
                        last_finalized_id = DESC0_ID(desc_ring->count_bits) + 1;
                } else {
                        /* Record must have been overwritten. Try again. */
                        goto try_again;
                }
        }

        /* Diff of known descriptor IDs to compute related sequence numbers. */
        diff = head_id - last_finalized_id;

        /*
         * @head_id points to the most recently reserved record, but this
         * function returns the sequence number that will be assigned to the
         * next (not yet reserved) record. Thus +1 is needed.
         */
        return (last_finalized_seq + diff + 1);
}

/*
 * Non-blocking read of a record.
 *
 * On success @seq is updated to the record that was read and (if provided)
 * @r and @line_count will contain the read/calculated data.
 *
 * On failure @seq is updated to a record that is not yet available to the
 * reader, but it will be the next record available to the reader.
 *
 * Note: When the current CPU is in panic, this function will skip over any
 *       non-existent/non-finalized records in order to allow the panic CPU
 *       to print any and all records that have been finalized.
 */
static bool _prb_read_valid(struct printk_ringbuffer *rb, u64 *seq,
                            struct printk_record *r, unsigned int *line_count)
{
        u64 tail_seq;
        int err;

        while ((err = prb_read(rb, *seq, r, line_count))) {
                tail_seq = prb_first_seq(rb);

                if (*seq < tail_seq) {
                        /*
                         * Behind the tail. Catch up and try again. This
                         * can happen for -ENOENT and -EINVAL cases.
                         */
                        *seq = tail_seq;

                } else if (err == -ENOENT) {
                        /* Record exists, but the data was lost. Skip. */
                        (*seq)++;

                } else {
                        /*
                         * Non-existent/non-finalized record. Must stop.
                         *
                         * For panic situations it cannot be expected that
                         * non-finalized records will become finalized. But
                         * there may be other finalized records beyond that
                         * need to be printed for a panic situation. If this
                         * is the panic CPU, skip this
                         * non-existent/non-finalized record unless non-panic
                         * CPUs are still running and their debugging is
                         * explicitly enabled.
                         *
                         * Note that new messages printed on panic CPU are
                         * finalized when we are here. The only exception
                         * might be the last message without trailing newline.
                         * But it would have the sequence number returned
                         * by "prb_next_reserve_seq() - 1".
                         */
                        if (this_cpu_in_panic() &&
                            (!debug_non_panic_cpus || legacy_allow_panic_sync) &&
                            ((*seq + 1) < prb_next_reserve_seq(rb))) {
                                (*seq)++;
                        } else {
                                return false;
                        }
                }
        }

        return true;
}

/**
 * prb_read_valid() - Non-blocking read of a requested record or (if gone)
 *                    the next available record.
 *
 * @rb:  The ringbuffer to read from.
 * @seq: The sequence number of the record to read.
 * @r:   A record data buffer to store the read record to.
 *
 * This is the public function available to readers to read a record.
 *
 * The reader provides the @info and @text_buf buffers of @r to be
 * filled in. Any of the buffer pointers can be set to NULL if the reader
 * is not interested in that data. To ensure proper initialization of @r,
 * prb_rec_init_rd() should be used.
 *
 * Context: Any context.
 * Return: true if a record was read, otherwise false.
 *
 * On success, the reader must check r->info.seq to see which record was
 * actually read. This allows the reader to detect dropped records.
 *
 * Failure means @seq refers to a record not yet available to the reader.
 */
bool prb_read_valid(struct printk_ringbuffer *rb, u64 seq,
                    struct printk_record *r)
{
        return _prb_read_valid(rb, &seq, r, NULL);
}

/**
 * prb_read_valid_info() - Non-blocking read of meta data for a requested
 *                         record or (if gone) the next available record.
 *
 * @rb:         The ringbuffer to read from.
 * @seq:        The sequence number of the record to read.
 * @info:       A buffer to store the read record meta data to.
 * @line_count: A buffer to store the number of lines in the record text.
 *
 * This is the public function available to readers to read only the
 * meta data of a record.
 *
 * The reader provides the @info, @line_count buffers to be filled in.
 * Either of the buffer pointers can be set to NULL if the reader is not
 * interested in that data.
 *
 * Context: Any context.
 * Return: true if a record's meta data was read, otherwise false.
 *
 * On success, the reader must check info->seq to see which record meta data
 * was actually read. This allows the reader to detect dropped records.
 *
 * Failure means @seq refers to a record not yet available to the reader.
 */
bool prb_read_valid_info(struct printk_ringbuffer *rb, u64 seq,
                         struct printk_info *info, unsigned int *line_count)
{
        struct printk_record r;

        prb_rec_init_rd(&r, info, NULL, 0);

        return _prb_read_valid(rb, &seq, &r, line_count);
}

/**
 * prb_first_valid_seq() - Get the sequence number of the oldest available
 *                         record.
 *
 * @rb: The ringbuffer to get the sequence number from.
 *
 * This is the public function available to readers to see what the
 * first/oldest valid sequence number is.
 *
 * This provides readers a starting point to begin iterating the ringbuffer.
 *
 * Context: Any context.
 * Return: The sequence number of the first/oldest record or, if the
 *         ringbuffer is empty, 0 is returned.
 */
u64 prb_first_valid_seq(struct printk_ringbuffer *rb)
{
        u64 seq = 0;

        if (!_prb_read_valid(rb, &seq, NULL, NULL))
                return 0;

        return seq;
}

/**
 * prb_next_seq() - Get the sequence number after the last available record.
 *
 * @rb:  The ringbuffer to get the sequence number from.
 *
 * This is the public function available to readers to see what the next
 * newest sequence number available to readers will be.
 *
 * This provides readers a sequence number to jump to if all currently
 * available records should be skipped. It is guaranteed that all records
 * previous to the returned value have been finalized and are (or were)
 * available to the reader.
 *
 * Context: Any context.
 * Return: The sequence number of the next newest (not yet available) record
 *         for readers.
 */
u64 prb_next_seq(struct printk_ringbuffer *rb)
{
        u64 seq;

        seq = desc_last_finalized_seq(rb);

        /*
         * Begin searching after the last finalized record.
         *
         * On 0, the search must begin at 0 because of hack#2
         * of the bootstrapping phase it is not known if a
         * record at index 0 exists.
         */
        if (seq != 0)
                seq++;

        /*
         * The information about the last finalized @seq might be inaccurate.
         * Search forward to find the current one.
         */
        while (_prb_read_valid(rb, &seq, NULL, NULL))
                seq++;

        return seq;
}

/**
 * prb_init() - Initialize a ringbuffer to use provided external buffers.
 *
 * @rb:       The ringbuffer to initialize.
 * @text_buf: The data buffer for text data.
 * @textbits: The size of @text_buf as a power-of-2 value.
 * @descs:    The descriptor buffer for ringbuffer records.
 * @descbits: The count of @descs items as a power-of-2 value.
 * @infos:    The printk_info buffer for ringbuffer records.
 *
 * This is the public function available to writers to setup a ringbuffer
 * during runtime using provided buffers.
 *
 * This must match the initialization of DEFINE_PRINTKRB().
 *
 * Context: Any context.
 */
void prb_init(struct printk_ringbuffer *rb,
              char *text_buf, unsigned int textbits,
              struct prb_desc *descs, unsigned int descbits,
              struct printk_info *infos)
{
        memset(descs, 0, _DESCS_COUNT(descbits) * sizeof(descs[0]));
        memset(infos, 0, _DESCS_COUNT(descbits) * sizeof(infos[0]));

        rb->desc_ring.count_bits = descbits;
        rb->desc_ring.descs = descs;
        rb->desc_ring.infos = infos;
        atomic_long_set(&rb->desc_ring.head_id, DESC0_ID(descbits));
        atomic_long_set(&rb->desc_ring.tail_id, DESC0_ID(descbits));
        atomic_long_set(&rb->desc_ring.last_finalized_seq, 0);

        rb->text_data_ring.size_bits = textbits;
        rb->text_data_ring.data = text_buf;
        atomic_long_set(&rb->text_data_ring.head_lpos, BLK0_LPOS(textbits));
        atomic_long_set(&rb->text_data_ring.tail_lpos, BLK0_LPOS(textbits));

        atomic_long_set(&rb->fail, 0);

        atomic_long_set(&(descs[_DESCS_COUNT(descbits) - 1].state_var), DESC0_SV(descbits));
        descs[_DESCS_COUNT(descbits) - 1].text_blk_lpos.begin = FAILED_LPOS;
        descs[_DESCS_COUNT(descbits) - 1].text_blk_lpos.next = FAILED_LPOS;

        infos[0].seq = -(u64)_DESCS_COUNT(descbits);
        infos[_DESCS_COUNT(descbits) - 1].seq = 0;
}

/**
 * prb_record_text_space() - Query the full actual used ringbuffer space for
 *                           the text data of a reserved entry.
 *
 * @e: The successfully reserved entry to query.
 *
 * This is the public function available to writers to see how much actual
 * space is used in the ringbuffer to store the text data of the specified
 * entry.
 *
 * This function is only valid if @e has been successfully reserved using
 * prb_reserve().
 *
 * Context: Any context.
 * Return: The size in bytes used by the text data of the associated record.
 */
unsigned int prb_record_text_space(struct prb_reserved_entry *e)
{
        return e->text_space;
}




























































































 2056 






















 2054 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Derived from arch/ppc/mm/extable.c and arch/i386/mm/extable.c.
 *
 * Copyright (C) 2004 Paul Mackerras, IBM Corp.
 */

#include <linux/bsearch.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/sort.h>
#include <linux/uaccess.h>
#include <linux/extable.h>

#ifndef ARCH_HAS_RELATIVE_EXTABLE
#define ex_to_insn(x)        ((x)->insn)
#else
static inline unsigned long ex_to_insn(const struct exception_table_entry *x)
{
        return (unsigned long)&x->insn + x->insn;
}
#endif

#ifndef ARCH_HAS_RELATIVE_EXTABLE
#define swap_ex                NULL
#else
static void swap_ex(void *a, void *b, int size)
{
        struct exception_table_entry *x = a, *y = b, tmp;
        int delta = b - a;

        tmp = *x;
        x->insn = y->insn + delta;
        y->insn = tmp.insn - delta;

#ifdef swap_ex_entry_fixup
        swap_ex_entry_fixup(x, y, tmp, delta);
#else
        x->fixup = y->fixup + delta;
        y->fixup = tmp.fixup - delta;
#endif
}
#endif /* ARCH_HAS_RELATIVE_EXTABLE */

/*
 * The exception table needs to be sorted so that the binary
 * search that we use to find entries in it works properly.
 * This is used both for the kernel exception table and for
 * the exception tables of modules that get loaded.
 */
static int cmp_ex_sort(const void *a, const void *b)
{
        const struct exception_table_entry *x = a, *y = b;

        /* avoid overflow */
        if (ex_to_insn(x) > ex_to_insn(y))
                return 1;
        if (ex_to_insn(x) < ex_to_insn(y))
                return -1;
        return 0;
}

void sort_extable(struct exception_table_entry *start,
                  struct exception_table_entry *finish)
{
        sort(start, finish - start, sizeof(struct exception_table_entry),
             cmp_ex_sort, swap_ex);
}

#ifdef CONFIG_MODULES
/*
 * If the exception table is sorted, any referring to the module init
 * will be at the beginning or the end.
 */
void trim_init_extable(struct module *m)
{
        /*trim the beginning*/
        while (m->num_exentries &&
               within_module_init(ex_to_insn(&m->extable[0]), m)) {
                m->extable++;
                m->num_exentries--;
        }
        /*trim the end*/
        while (m->num_exentries &&
               within_module_init(ex_to_insn(&m->extable[m->num_exentries - 1]),
                                  m))
                m->num_exentries--;
}
#endif /* CONFIG_MODULES */

static int cmp_ex_search(const void *key, const void *elt)
{
        const struct exception_table_entry *_elt = elt;
        unsigned long _key = *(unsigned long *)key;

        /* avoid overflow */
        if (_key > ex_to_insn(_elt))
                return 1;
        if (_key < ex_to_insn(_elt))
                return -1;
        return 0;
}

/*
 * Search one exception table for an entry corresponding to the
 * given instruction address, and return the address of the entry,
 * or NULL if none is found.
 * We use a binary search, and thus we assume that the table is
 * already sorted.
 */
const struct exception_table_entry *
search_extable(const struct exception_table_entry *base,
               const size_t num,
               unsigned long value)
{
        return bsearch(&value, base, num,
                       sizeof(struct exception_table_entry), cmp_ex_search);
}











































































































































































































































































































































































































































































































































































































































































































































































































































































  106 
  248 

  106 












  246 




    8 



  237 



  238 











































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Helpers for formatting and printing strings
 *
 * Copyright 31 August 2008 James Bottomley
 * Copyright (C) 2013, Intel Corporation
 */
#include <linux/bug.h>
#include <linux/kernel.h>
#include <linux/math64.h>
#include <linux/export.h>
#include <linux/ctype.h>
#include <linux/device.h>
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/limits.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/string_helpers.h>
#include <kunit/test.h>
#include <kunit/test-bug.h>

/**
 * string_get_size - get the size in the specified units
 * @size:        The size to be converted in blocks
 * @blk_size:        Size of the block (use 1 for size in bytes)
 * @units:        Units to use (powers of 1000 or 1024), whether to include space separator
 * @buf:        buffer to format to
 * @len:        length of buffer
 *
 * This function returns a string formatted to 3 significant figures
 * giving the size in the required units.  @buf should have room for
 * at least 9 bytes and will always be zero terminated.
 *
 * Return value: number of characters of output that would have been written
 * (which may be greater than len, if output was truncated).
 */
int string_get_size(u64 size, u64 blk_size, const enum string_size_units units,
                    char *buf, int len)
{
        enum string_size_units units_base = units & STRING_UNITS_MASK;
        static const char *const units_10[] = {
                "", "k", "M", "G", "T", "P", "E", "Z", "Y",
        };
        static const char *const units_2[] = {
                "", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi",
        };
        static const char *const *const units_str[] = {
                [STRING_UNITS_10] = units_10,
                [STRING_UNITS_2] = units_2,
        };
        static const unsigned int divisor[] = {
                [STRING_UNITS_10] = 1000,
                [STRING_UNITS_2] = 1024,
        };
        static const unsigned int rounding[] = { 500, 50, 5 };
        int i = 0, j;
        u32 remainder = 0, sf_cap;
        char tmp[12];
        const char *unit;

        tmp[0] = '\0';

        if (blk_size == 0)
                size = 0;
        if (size == 0)
                goto out;

        /* This is Napier's algorithm.  Reduce the original block size to
         *
         * coefficient * divisor[units_base]^i
         *
         * we do the reduction so both coefficients are just under 32 bits so
         * that multiplying them together won't overflow 64 bits and we keep
         * as much precision as possible in the numbers.
         *
         * Note: it's safe to throw away the remainders here because all the
         * precision is in the coefficients.
         */
        while (blk_size >> 32) {
                do_div(blk_size, divisor[units_base]);
                i++;
        }

        while (size >> 32) {
                do_div(size, divisor[units_base]);
                i++;
        }

        /* now perform the actual multiplication keeping i as the sum of the
         * two logarithms */
        size *= blk_size;

        /* and logarithmically reduce it until it's just under the divisor */
        while (size >= divisor[units_base]) {
                remainder = do_div(size, divisor[units_base]);
                i++;
        }

        /* work out in j how many digits of precision we need from the
         * remainder */
        sf_cap = size;
        for (j = 0; sf_cap*10 < 1000; j++)
                sf_cap *= 10;

        if (units_base == STRING_UNITS_2) {
                /* express the remainder as a decimal.  It's currently the
                 * numerator of a fraction whose denominator is
                 * divisor[units_base], which is 1 << 10 for STRING_UNITS_2 */
                remainder *= 1000;
                remainder >>= 10;
        }

        /* add a 5 to the digit below what will be printed to ensure
         * an arithmetical round up and carry it through to size */
        remainder += rounding[j];
        if (remainder >= 1000) {
                remainder -= 1000;
                size += 1;
        }

        if (j) {
                snprintf(tmp, sizeof(tmp), ".%03u", remainder);
                tmp[j+1] = '\0';
        }

 out:
        if (i >= ARRAY_SIZE(units_2))
                unit = "UNK";
        else
                unit = units_str[units_base][i];

        return snprintf(buf, len, "%u%s%s%s%s", (u32)size, tmp,
                        (units & STRING_UNITS_NO_SPACE) ? "" : " ",
                        unit,
                        (units & STRING_UNITS_NO_BYTES) ? "" : "B");
}
EXPORT_SYMBOL(string_get_size);

/**
 * parse_int_array_user - Split string into a sequence of integers
 * @from:        The user space buffer to read from
 * @count:        The maximum number of bytes to read
 * @array:        Returned pointer to sequence of integers
 *
 * On success @array is allocated and initialized with a sequence of
 * integers extracted from the @from plus an additional element that
 * begins the sequence and specifies the integers count.
 *
 * Caller takes responsibility for freeing @array when it is no longer
 * needed.
 */
int parse_int_array_user(const char __user *from, size_t count, int **array)
{
        int *ints, nints;
        char *buf;
        int ret = 0;

        buf = memdup_user_nul(from, count);
        if (IS_ERR(buf))
                return PTR_ERR(buf);

        get_options(buf, 0, &nints);
        if (!nints) {
                ret = -ENOENT;
                goto free_buf;
        }

        ints = kcalloc(nints + 1, sizeof(*ints), GFP_KERNEL);
        if (!ints) {
                ret = -ENOMEM;
                goto free_buf;
        }

        get_options(buf, nints + 1, ints);
        *array = ints;

free_buf:
        kfree(buf);
        return ret;
}
EXPORT_SYMBOL(parse_int_array_user);

static bool unescape_space(char **src, char **dst)
{
        char *p = *dst, *q = *src;

        switch (*q) {
        case 'n':
                *p = '\n';
                break;
        case 'r':
                *p = '\r';
                break;
        case 't':
                *p = '\t';
                break;
        case 'v':
                *p = '\v';
                break;
        case 'f':
                *p = '\f';
                break;
        default:
                return false;
        }
        *dst += 1;
        *src += 1;
        return true;
}

static bool unescape_octal(char **src, char **dst)
{
        char *p = *dst, *q = *src;
        u8 num;

        if (isodigit(*q) == 0)
                return false;

        num = (*q++) & 7;
        while (num < 32 && isodigit(*q) && (q - *src < 3)) {
                num <<= 3;
                num += (*q++) & 7;
        }
        *p = num;
        *dst += 1;
        *src = q;
        return true;
}

static bool unescape_hex(char **src, char **dst)
{
        char *p = *dst, *q = *src;
        int digit;
        u8 num;

        if (*q++ != 'x')
                return false;

        num = digit = hex_to_bin(*q++);
        if (digit < 0)
                return false;

        digit = hex_to_bin(*q);
        if (digit >= 0) {
                q++;
                num = (num << 4) | digit;
        }
        *p = num;
        *dst += 1;
        *src = q;
        return true;
}

static bool unescape_special(char **src, char **dst)
{
        char *p = *dst, *q = *src;

        switch (*q) {
        case '\"':
                *p = '\"';
                break;
        case '\\':
                *p = '\\';
                break;
        case 'a':
                *p = '\a';
                break;
        case 'e':
                *p = '\e';
                break;
        default:
                return false;
        }
        *dst += 1;
        *src += 1;
        return true;
}

/**
 * string_unescape - unquote characters in the given string
 * @src:        source buffer (escaped)
 * @dst:        destination buffer (unescaped)
 * @size:        size of the destination buffer (0 to unlimit)
 * @flags:        combination of the flags.
 *
 * Description:
 * The function unquotes characters in the given string.
 *
 * Because the size of the output will be the same as or less than the size of
 * the input, the transformation may be performed in place.
 *
 * Caller must provide valid source and destination pointers. Be aware that
 * destination buffer will always be NULL-terminated. Source string must be
 * NULL-terminated as well.  The supported flags are::
 *
 *        UNESCAPE_SPACE:
 *                '\f' - form feed
 *                '\n' - new line
 *                '\r' - carriage return
 *                '\t' - horizontal tab
 *                '\v' - vertical tab
 *        UNESCAPE_OCTAL:
 *                '\NNN' - byte with octal value NNN (1 to 3 digits)
 *        UNESCAPE_HEX:
 *                '\xHH' - byte with hexadecimal value HH (1 to 2 digits)
 *        UNESCAPE_SPECIAL:
 *                '\"' - double quote
 *                '\\' - backslash
 *                '\a' - alert (BEL)
 *                '\e' - escape
 *        UNESCAPE_ANY:
 *                all previous together
 *
 * Return:
 * The amount of the characters processed to the destination buffer excluding
 * trailing '\0' is returned.
 */
int string_unescape(char *src, char *dst, size_t size, unsigned int flags)
{
        char *out = dst;

        if (!size)
                size = SIZE_MAX;

        while (*src && --size) {
                if (src[0] == '\\' && src[1] != '\0' && size > 1) {
                        src++;
                        size--;

                        if (flags & UNESCAPE_SPACE &&
                                        unescape_space(&src, &out))
                                continue;

                        if (flags & UNESCAPE_OCTAL &&
                                        unescape_octal(&src, &out))
                                continue;

                        if (flags & UNESCAPE_HEX &&
                                        unescape_hex(&src, &out))
                                continue;

                        if (flags & UNESCAPE_SPECIAL &&
                                        unescape_special(&src, &out))
                                continue;

                        *out++ = '\\';
                }
                *out++ = *src++;
        }
        *out = '\0';

        return out - dst;
}
EXPORT_SYMBOL(string_unescape);

static bool escape_passthrough(unsigned char c, char **dst, char *end)
{
        char *out = *dst;

        if (out < end)
                *out = c;
        *dst = out + 1;
        return true;
}

static bool escape_space(unsigned char c, char **dst, char *end)
{
        char *out = *dst;
        unsigned char to;

        switch (c) {
        case '\n':
                to = 'n';
                break;
        case '\r':
                to = 'r';
                break;
        case '\t':
                to = 't';
                break;
        case '\v':
                to = 'v';
                break;
        case '\f':
                to = 'f';
                break;
        default:
                return false;
        }

        if (out < end)
                *out = '\\';
        ++out;
        if (out < end)
                *out = to;
        ++out;

        *dst = out;
        return true;
}

static bool escape_special(unsigned char c, char **dst, char *end)
{
        char *out = *dst;
        unsigned char to;

        switch (c) {
        case '\\':
                to = '\\';
                break;
        case '\a':
                to = 'a';
                break;
        case '\e':
                to = 'e';
                break;
        case '"':
                to = '"';
                break;
        default:
                return false;
        }

        if (out < end)
                *out = '\\';
        ++out;
        if (out < end)
                *out = to;
        ++out;

        *dst = out;
        return true;
}

static bool escape_null(unsigned char c, char **dst, char *end)
{
        char *out = *dst;

        if (c)
                return false;

        if (out < end)
                *out = '\\';
        ++out;
        if (out < end)
                *out = '0';
        ++out;

        *dst = out;
        return true;
}

static bool escape_octal(unsigned char c, char **dst, char *end)
{
        char *out = *dst;

        if (out < end)
                *out = '\\';
        ++out;
        if (out < end)
                *out = ((c >> 6) & 0x07) + '0';
        ++out;
        if (out < end)
                *out = ((c >> 3) & 0x07) + '0';
        ++out;
        if (out < end)
                *out = ((c >> 0) & 0x07) + '0';
        ++out;

        *dst = out;
        return true;
}

static bool escape_hex(unsigned char c, char **dst, char *end)
{
        char *out = *dst;

        if (out < end)
                *out = '\\';
        ++out;
        if (out < end)
                *out = 'x';
        ++out;
        if (out < end)
                *out = hex_asc_hi(c);
        ++out;
        if (out < end)
                *out = hex_asc_lo(c);
        ++out;

        *dst = out;
        return true;
}

/**
 * string_escape_mem - quote characters in the given memory buffer
 * @src:        source buffer (unescaped)
 * @isz:        source buffer size
 * @dst:        destination buffer (escaped)
 * @osz:        destination buffer size
 * @flags:        combination of the flags
 * @only:        NULL-terminated string containing characters used to limit
 *                the selected escape class. If characters are included in @only
 *                that would not normally be escaped by the classes selected
 *                in @flags, they will be copied to @dst unescaped.
 *
 * Description:
 * The process of escaping byte buffer includes several parts. They are applied
 * in the following sequence.
 *
 *        1. The character is not matched to the one from @only string and thus
 *           must go as-is to the output.
 *        2. The character is matched to the printable and ASCII classes, if asked,
 *           and in case of match it passes through to the output.
 *        3. The character is matched to the printable or ASCII class, if asked,
 *           and in case of match it passes through to the output.
 *        4. The character is checked if it falls into the class given by @flags.
 *           %ESCAPE_OCTAL and %ESCAPE_HEX are going last since they cover any
 *           character. Note that they actually can't go together, otherwise
 *           %ESCAPE_HEX will be ignored.
 *
 * Caller must provide valid source and destination pointers. Be aware that
 * destination buffer will not be NULL-terminated, thus caller have to append
 * it if needs. The supported flags are::
 *
 *        %ESCAPE_SPACE: (special white space, not space itself)
 *                '\f' - form feed
 *                '\n' - new line
 *                '\r' - carriage return
 *                '\t' - horizontal tab
 *                '\v' - vertical tab
 *        %ESCAPE_SPECIAL:
 *                '\"' - double quote
 *                '\\' - backslash
 *                '\a' - alert (BEL)
 *                '\e' - escape
 *        %ESCAPE_NULL:
 *                '\0' - null
 *        %ESCAPE_OCTAL:
 *                '\NNN' - byte with octal value NNN (3 digits)
 *        %ESCAPE_ANY:
 *                all previous together
 *        %ESCAPE_NP:
 *                escape only non-printable characters, checked by isprint()
 *        %ESCAPE_ANY_NP:
 *                all previous together
 *        %ESCAPE_HEX:
 *                '\xHH' - byte with hexadecimal value HH (2 digits)
 *        %ESCAPE_NA:
 *                escape only non-ascii characters, checked by isascii()
 *        %ESCAPE_NAP:
 *                escape only non-printable or non-ascii characters
 *        %ESCAPE_APPEND:
 *                append characters from @only to be escaped by the given classes
 *
 * %ESCAPE_APPEND would help to pass additional characters to the escaped, when
 * one of %ESCAPE_NP, %ESCAPE_NA, or %ESCAPE_NAP is provided.
 *
 * One notable caveat, the %ESCAPE_NAP, %ESCAPE_NP and %ESCAPE_NA have the
 * higher priority than the rest of the flags (%ESCAPE_NAP is the highest).
 * It doesn't make much sense to use either of them without %ESCAPE_OCTAL
 * or %ESCAPE_HEX, because they cover most of the other character classes.
 * %ESCAPE_NAP can utilize %ESCAPE_SPACE or %ESCAPE_SPECIAL in addition to
 * the above.
 *
 * Return:
 * The total size of the escaped output that would be generated for
 * the given input and flags. To check whether the output was
 * truncated, compare the return value to osz. There is room left in
 * dst for a '\0' terminator if and only if ret < osz.
 */
int string_escape_mem(const char *src, size_t isz, char *dst, size_t osz,
                      unsigned int flags, const char *only)
{
        char *p = dst;
        char *end = p + osz;
        bool is_dict = only && *only;
        bool is_append = flags & ESCAPE_APPEND;

        while (isz--) {
                unsigned char c = *src++;
                bool in_dict = is_dict && strchr(only, c);

                /*
                 * Apply rules in the following sequence:
                 *        - the @only string is supplied and does not contain a
                 *          character under question
                 *        - the character is printable and ASCII, when @flags has
                 *          %ESCAPE_NAP bit set
                 *        - the character is printable, when @flags has
                 *          %ESCAPE_NP bit set
                 *        - the character is ASCII, when @flags has
                 *          %ESCAPE_NA bit set
                 *        - the character doesn't fall into a class of symbols
                 *          defined by given @flags
                 * In these cases we just pass through a character to the
                 * output buffer.
                 *
                 * When %ESCAPE_APPEND is passed, the characters from @only
                 * have been excluded from the %ESCAPE_NAP, %ESCAPE_NP, and
                 * %ESCAPE_NA cases.
                 */
                if (!(is_append || in_dict) && is_dict &&
                                          escape_passthrough(c, &p, end))
                        continue;

                if (!(is_append && in_dict) && isascii(c) && isprint(c) &&
                    flags & ESCAPE_NAP && escape_passthrough(c, &p, end))
                        continue;

                if (!(is_append && in_dict) && isprint(c) &&
                    flags & ESCAPE_NP && escape_passthrough(c, &p, end))
                        continue;

                if (!(is_append && in_dict) && isascii(c) &&
                    flags & ESCAPE_NA && escape_passthrough(c, &p, end))
                        continue;

                if (flags & ESCAPE_SPACE && escape_space(c, &p, end))
                        continue;

                if (flags & ESCAPE_SPECIAL && escape_special(c, &p, end))
                        continue;

                if (flags & ESCAPE_NULL && escape_null(c, &p, end))
                        continue;

                /* ESCAPE_OCTAL and ESCAPE_HEX always go last */
                if (flags & ESCAPE_OCTAL && escape_octal(c, &p, end))
                        continue;

                if (flags & ESCAPE_HEX && escape_hex(c, &p, end))
                        continue;

                escape_passthrough(c, &p, end);
        }

        return p - dst;
}
EXPORT_SYMBOL(string_escape_mem);

/*
 * Return an allocated string that has been escaped of special characters
 * and double quotes, making it safe to log in quotes.
 */
char *kstrdup_quotable(const char *src, gfp_t gfp)
{
        size_t slen, dlen;
        char *dst;
        const int flags = ESCAPE_HEX;
        const char esc[] = "\f\n\r\t\v\a\e\\\"";

        if (!src)
                return NULL;
        slen = strlen(src);

        dlen = string_escape_mem(src, slen, NULL, 0, flags, esc);
        dst = kmalloc(dlen + 1, gfp);
        if (!dst)
                return NULL;

        WARN_ON(string_escape_mem(src, slen, dst, dlen, flags, esc) != dlen);
        dst[dlen] = '\0';

        return dst;
}
EXPORT_SYMBOL_GPL(kstrdup_quotable);

/*
 * Returns allocated NULL-terminated string containing process
 * command line, with inter-argument NULLs replaced with spaces,
 * and other special characters escaped.
 */
char *kstrdup_quotable_cmdline(struct task_struct *task, gfp_t gfp)
{
        char *buffer, *quoted;
        int i, res;

        buffer = kmalloc(PAGE_SIZE, GFP_KERNEL);
        if (!buffer)
                return NULL;

        res = get_cmdline(task, buffer, PAGE_SIZE - 1);
        buffer[res] = '\0';

        /* Collapse trailing NULLs, leave res pointing to last non-NULL. */
        while (--res >= 0 && buffer[res] == '\0')
                ;

        /* Replace inter-argument NULLs. */
        for (i = 0; i <= res; i++)
                if (buffer[i] == '\0')
                        buffer[i] = ' ';

        /* Make sure result is printable. */
        quoted = kstrdup_quotable(buffer, gfp);
        kfree(buffer);
        return quoted;
}
EXPORT_SYMBOL_GPL(kstrdup_quotable_cmdline);

/*
 * Returns allocated NULL-terminated string containing pathname,
 * with special characters escaped, able to be safely logged. If
 * there is an error, the leading character will be "<".
 */
char *kstrdup_quotable_file(struct file *file, gfp_t gfp)
{
        char *temp, *pathname;

        if (!file)
                return kstrdup("<unknown>", gfp);

        /* We add 11 spaces for ' (deleted)' to be appended */
        temp = kmalloc(PATH_MAX + 11, GFP_KERNEL);
        if (!temp)
                return kstrdup("<no_memory>", gfp);

        pathname = file_path(file, temp, PATH_MAX + 11);
        if (IS_ERR(pathname))
                pathname = kstrdup("<too_long>", gfp);
        else
                pathname = kstrdup_quotable(pathname, gfp);

        kfree(temp);
        return pathname;
}
EXPORT_SYMBOL_GPL(kstrdup_quotable_file);

/*
 * Returns duplicate string in which the @old characters are replaced by @new.
 */
char *kstrdup_and_replace(const char *src, char old, char new, gfp_t gfp)
{
        char *dst;

        dst = kstrdup(src, gfp);
        if (!dst)
                return NULL;

        return strreplace(dst, old, new);
}
EXPORT_SYMBOL_GPL(kstrdup_and_replace);

/**
 * kasprintf_strarray - allocate and fill array of sequential strings
 * @gfp: flags for the slab allocator
 * @prefix: prefix to be used
 * @n: amount of lines to be allocated and filled
 *
 * Allocates and fills @n strings using pattern "%s-%zu", where prefix
 * is provided by caller. The caller is responsible to free them with
 * kfree_strarray() after use.
 *
 * Returns array of strings or NULL when memory can't be allocated.
 */
char **kasprintf_strarray(gfp_t gfp, const char *prefix, size_t n)
{
        char **names;
        size_t i;

        names = kcalloc(n + 1, sizeof(char *), gfp);
        if (!names)
                return NULL;

        for (i = 0; i < n; i++) {
                names[i] = kasprintf(gfp, "%s-%zu", prefix, i);
                if (!names[i]) {
                        kfree_strarray(names, i);
                        return NULL;
                }
        }

        return names;
}
EXPORT_SYMBOL_GPL(kasprintf_strarray);

/**
 * kfree_strarray - free a number of dynamically allocated strings contained
 *                  in an array and the array itself
 *
 * @array: Dynamically allocated array of strings to free.
 * @n: Number of strings (starting from the beginning of the array) to free.
 *
 * Passing a non-NULL @array and @n == 0 as well as NULL @array are valid
 * use-cases. If @array is NULL, the function does nothing.
 */
void kfree_strarray(char **array, size_t n)
{
        unsigned int i;

        if (!array)
                return;

        for (i = 0; i < n; i++)
                kfree(array[i]);
        kfree(array);
}
EXPORT_SYMBOL_GPL(kfree_strarray);

struct strarray {
        char **array;
        size_t n;
};

static void devm_kfree_strarray(struct device *dev, void *res)
{
        struct strarray *array = res;

        kfree_strarray(array->array, array->n);
}

char **devm_kasprintf_strarray(struct device *dev, const char *prefix, size_t n)
{
        struct strarray *ptr;

        ptr = devres_alloc(devm_kfree_strarray, sizeof(*ptr), GFP_KERNEL);
        if (!ptr)
                return ERR_PTR(-ENOMEM);

        ptr->array = kasprintf_strarray(GFP_KERNEL, prefix, n);
        if (!ptr->array) {
                devres_free(ptr);
                return ERR_PTR(-ENOMEM);
        }

        ptr->n = n;
        devres_add(dev, ptr);

        return ptr->array;
}
EXPORT_SYMBOL_GPL(devm_kasprintf_strarray);

/**
 * skip_spaces - Removes leading whitespace from @str.
 * @str: The string to be stripped.
 *
 * Returns a pointer to the first non-whitespace character in @str.
 */
char *skip_spaces(const char *str)
{
        while (isspace(*str))
                ++str;
        return (char *)str;
}
EXPORT_SYMBOL(skip_spaces);

/**
 * strim - Removes leading and trailing whitespace from @s.
 * @s: The string to be stripped.
 *
 * Note that the first trailing whitespace is replaced with a %NUL-terminator
 * in the given string @s. Returns a pointer to the first non-whitespace
 * character in @s.
 */
char *strim(char *s)
{
        size_t size;
        char *end;

        size = strlen(s);
        if (!size)
                return s;

        end = s + size - 1;
        while (end >= s && isspace(*end))
                end--;
        *(end + 1) = '\0';

        return skip_spaces(s);
}
EXPORT_SYMBOL(strim);

/**
 * sysfs_streq - return true if strings are equal, modulo trailing newline
 * @s1: one string
 * @s2: another string
 *
 * This routine returns true iff two strings are equal, treating both
 * NUL and newline-then-NUL as equivalent string terminations.  It's
 * geared for use with sysfs input strings, which generally terminate
 * with newlines but are compared against values without newlines.
 */
bool sysfs_streq(const char *s1, const char *s2)
{
        while (*s1 && *s1 == *s2) {
                s1++;
                s2++;
        }

        if (*s1 == *s2)
                return true;
        if (!*s1 && *s2 == '\n' && !s2[1])
                return true;
        if (*s1 == '\n' && !s1[1] && !*s2)
                return true;
        return false;
}
EXPORT_SYMBOL(sysfs_streq);

/**
 * match_string - matches given string in an array
 * @array:        array of strings
 * @n:                number of strings in the array or -1 for NULL terminated arrays
 * @string:        string to match with
 *
 * This routine will look for a string in an array of strings up to the
 * n-th element in the array or until the first NULL element.
 *
 * Historically the value of -1 for @n, was used to search in arrays that
 * are NULL terminated. However, the function does not make a distinction
 * when finishing the search: either @n elements have been compared OR
 * the first NULL element was found.
 *
 * Return:
 * index of a @string in the @array if matches, or %-EINVAL otherwise.
 */
int match_string(const char * const *array, size_t n, const char *string)
{
        int index;
        const char *item;

        for (index = 0; index < n; index++) {
                item = array[index];
                if (!item)
                        break;
                if (!strcmp(item, string))
                        return index;
        }

        return -EINVAL;
}
EXPORT_SYMBOL(match_string);

/**
 * __sysfs_match_string - matches given string in an array
 * @array: array of strings
 * @n: number of strings in the array or -1 for NULL terminated arrays
 * @str: string to match with
 *
 * Returns index of @str in the @array or -EINVAL, just like match_string().
 * Uses sysfs_streq instead of strcmp for matching.
 *
 * This routine will look for a string in an array of strings up to the
 * n-th element in the array or until the first NULL element.
 *
 * Historically the value of -1 for @n, was used to search in arrays that
 * are NULL terminated. However, the function does not make a distinction
 * when finishing the search: either @n elements have been compared OR
 * the first NULL element was found.
 */
int __sysfs_match_string(const char * const *array, size_t n, const char *str)
{
        const char *item;
        int index;

        for (index = 0; index < n; index++) {
                item = array[index];
                if (!item)
                        break;
                if (sysfs_streq(item, str))
                        return index;
        }

        return -EINVAL;
}
EXPORT_SYMBOL(__sysfs_match_string);

/**
 * strreplace - Replace all occurrences of character in string.
 * @str: The string to operate on.
 * @old: The character being replaced.
 * @new: The character @old is replaced with.
 *
 * Replaces the each @old character with a @new one in the given string @str.
 *
 * Return: pointer to the string @str itself.
 */
char *strreplace(char *str, char old, char new)
{
        char *s = str;

        for (; *s; ++s)
                if (*s == old)
                        *s = new;
        return str;
}
EXPORT_SYMBOL(strreplace);

/**
 * memcpy_and_pad - Copy one buffer to another with padding
 * @dest: Where to copy to
 * @dest_len: The destination buffer size
 * @src: Where to copy from
 * @count: The number of bytes to copy
 * @pad: Character to use for padding if space is left in destination.
 */
void memcpy_and_pad(void *dest, size_t dest_len, const void *src, size_t count,
                    int pad)
{
        if (dest_len > count) {
                memcpy(dest, src, count);
                memset(dest + count, pad,  dest_len - count);
        } else {
                memcpy(dest, src, dest_len);
        }
}
EXPORT_SYMBOL(memcpy_and_pad);

#ifdef CONFIG_FORTIFY_SOURCE
/* These are placeholders for fortify compile-time warnings. */
void __read_overflow2_field(size_t avail, size_t wanted) { }
EXPORT_SYMBOL(__read_overflow2_field);
void __write_overflow_field(size_t avail, size_t wanted) { }
EXPORT_SYMBOL(__write_overflow_field);

static const char * const fortify_func_name[] = {
#define MAKE_FORTIFY_FUNC_NAME(func)        [MAKE_FORTIFY_FUNC(func)] = #func
        EACH_FORTIFY_FUNC(MAKE_FORTIFY_FUNC_NAME)
#undef  MAKE_FORTIFY_FUNC_NAME
};

void __fortify_report(const u8 reason, const size_t avail, const size_t size)
{
        const u8 func = FORTIFY_REASON_FUNC(reason);
        const bool write = FORTIFY_REASON_DIR(reason);
        const char *name;

        name = fortify_func_name[umin(func, FORTIFY_FUNC_UNKNOWN)];
        WARN(1, "%s: detected buffer overflow: %zu byte %s of buffer size %zu\n",
                 name, size, str_read_write(!write), avail);
}
EXPORT_SYMBOL(__fortify_report);

void __fortify_panic(const u8 reason, const size_t avail, const size_t size)
{
        __fortify_report(reason, avail, size);
        BUG();
}
EXPORT_SYMBOL(__fortify_panic);
#endif /* CONFIG_FORTIFY_SOURCE */



















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































  252 
















  685 



 1417 















   97 















  484 










 1336 









































 1919 



 1207 










 4714 





 3914 
 4390 











  342 








  263 










    6 








































 2250 



































    2 



























    3 



























































































  130 






   35 
  126 



























   23 





















































































   71 
  248 






   57 



  270 


















































  203 
































































 7492 
































  491 































   16 
   35 










   37 













 1365 






   32 




   15 
















  253 


































































































  126 
















   10 
 1711 
    8 






    7 














    4 

   32 















  173 


  109 








   10 












































    9 

  349 



   20 




































   27 






















   17 





























   25 





















    9 















    1 


























  107 



































































   12 














    3 
   62 














   16 















   38 
   52 




















    3 









   77 
































   77 












  776 


















 2575 
  361 















   91 













    7 






   87 
   16 





   87 






















  550 








    7 

































  550 












  549 
  548 





   31 
   63 





  488 




















  488 



















    7 






















 7262 




  165 




  165 
























 5304 
















 1167 
 1167 






























   29 









   23 














  670 

  685 







 4324 










 5489 




 5470 









 2695 


 2911 


  164 









   53 
 2941 




  246 
















  250 










 4167 



























  387 














































  298 











  215 



  627 






    1 











   11 






   11 






















































 1153 
  184 






 1296 






  718 





  958 






 6033 





  626 





 2305 
   41 



   12 
   15 




   59 












 5912 


















    4 


  370 

  228 






































   13 
  521 




  447 










  129 










  579 




  118 
























































 1493 
   46 




 1487 







  173 
  125 




    9 

  358 













    2 
    3 






   25 
   26 




   52 












  274 

  742 



 2336 














 4543 









  752 


















  993 




   11 
  977 



























































































































































































  232 









    6 









    1 



































































   16 



















   11 











































































    3 



















    3 
  311 















    7 


  319 



















   78 













   19 
  285 





































    1 



   16 
















   16 








  189 


  103 

   87 
    6 


















  113 











   34 




















    2 
    1 



















  289 

    2 
   14 








    5 
    5 















    7 


















    4 

















   24 






    1 
    1 
    2 





    3 
    3 
    6 















































































































































 3530 




    7 


















  170 


  270 







  273 
  441 
























 4412 

   90 



















   32 







   32 






















































   26 













  737 










  738 













  737 









  369 










  177 


  602 

































































































































































  661 
   16 















    2 
   44 
  175 
   15 





















    6 





   25 









   10 
   20 
   20 

   23 






   15 














   37 

   16 




































   21 








    5 















































































































































































 7386 
   24 







 4628 


   18 






  149 
  148 














   46 
   14 




 4711 
   18 









  239 



















    8 

  576 
























 4631 










  149 


  149 













































    1 
























   64 










 4582 







    1 
































































































   24 
   24 





























































































  211 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 *        Definitions for the 'struct sk_buff' memory handlers.
 *
 *        Authors:
 *                Alan Cox, <gw4pts@gw4pts.ampr.org>
 *                Florian La Roche, <rzsfl@rz.uni-sb.de>
 */

#ifndef _LINUX_SKBUFF_H
#define _LINUX_SKBUFF_H

#include <linux/kernel.h>
#include <linux/compiler.h>
#include <linux/time.h>
#include <linux/bug.h>
#include <linux/bvec.h>
#include <linux/cache.h>
#include <linux/rbtree.h>
#include <linux/socket.h>
#include <linux/refcount.h>

#include <linux/atomic.h>
#include <asm/types.h>
#include <linux/spinlock.h>
#include <net/checksum.h>
#include <linux/rcupdate.h>
#include <linux/dma-mapping.h>
#include <linux/netdev_features.h>
#include <net/flow_dissector.h>
#include <linux/in6.h>
#include <linux/if_packet.h>
#include <linux/llist.h>
#include <linux/page_frag_cache.h>
#include <net/flow.h>
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
#include <linux/netfilter/nf_conntrack_common.h>
#endif
#include <net/net_debug.h>
#include <net/dropreason-core.h>
#include <net/netmem.h>

/**
 * DOC: skb checksums
 *
 * The interface for checksum offload between the stack and networking drivers
 * is as follows...
 *
 * IP checksum related features
 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 *
 * Drivers advertise checksum offload capabilities in the features of a device.
 * From the stack's point of view these are capabilities offered by the driver.
 * A driver typically only advertises features that it is capable of offloading
 * to its device.
 *
 * .. flat-table:: Checksum related device features
 *   :widths: 1 10
 *
 *   * - %NETIF_F_HW_CSUM
 *     - The driver (or its device) is able to compute one
 *         IP (one's complement) checksum for any combination
 *         of protocols or protocol layering. The checksum is
 *         computed and set in a packet per the CHECKSUM_PARTIAL
 *         interface (see below).
 *
 *   * - %NETIF_F_IP_CSUM
 *     - Driver (device) is only able to checksum plain
 *         TCP or UDP packets over IPv4. These are specifically
 *         unencapsulated packets of the form IPv4|TCP or
 *         IPv4|UDP where the Protocol field in the IPv4 header
 *         is TCP or UDP. The IPv4 header may contain IP options.
 *         This feature cannot be set in features for a device
 *         with NETIF_F_HW_CSUM also set. This feature is being
 *         DEPRECATED (see below).
 *
 *   * - %NETIF_F_IPV6_CSUM
 *     - Driver (device) is only able to checksum plain
 *         TCP or UDP packets over IPv6. These are specifically
 *         unencapsulated packets of the form IPv6|TCP or
 *         IPv6|UDP where the Next Header field in the IPv6
 *         header is either TCP or UDP. IPv6 extension headers
 *         are not supported with this feature. This feature
 *         cannot be set in features for a device with
 *         NETIF_F_HW_CSUM also set. This feature is being
 *         DEPRECATED (see below).
 *
 *   * - %NETIF_F_RXCSUM
 *     - Driver (device) performs receive checksum offload.
 *         This flag is only used to disable the RX checksum
 *         feature for a device. The stack will accept receive
 *         checksum indication in packets received on a device
 *         regardless of whether NETIF_F_RXCSUM is set.
 *
 * Checksumming of received packets by device
 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 *
 * Indication of checksum verification is set in &sk_buff.ip_summed.
 * Possible values are:
 *
 * - %CHECKSUM_NONE
 *
 *   Device did not checksum this packet e.g. due to lack of capabilities.
 *   The packet contains full (though not verified) checksum in packet but
 *   not in skb->csum. Thus, skb->csum is undefined in this case.
 *
 * - %CHECKSUM_UNNECESSARY
 *
 *   The hardware you're dealing with doesn't calculate the full checksum
 *   (as in %CHECKSUM_COMPLETE), but it does parse headers and verify checksums
 *   for specific protocols. For such packets it will set %CHECKSUM_UNNECESSARY
 *   if their checksums are okay. &sk_buff.csum is still undefined in this case
 *   though. A driver or device must never modify the checksum field in the
 *   packet even if checksum is verified.
 *
 *   %CHECKSUM_UNNECESSARY is applicable to following protocols:
 *
 *     - TCP: IPv6 and IPv4.
 *     - UDP: IPv4 and IPv6. A device may apply CHECKSUM_UNNECESSARY to a
 *       zero UDP checksum for either IPv4 or IPv6, the networking stack
 *       may perform further validation in this case.
 *     - GRE: only if the checksum is present in the header.
 *     - SCTP: indicates the CRC in SCTP header has been validated.
 *     - FCOE: indicates the CRC in FC frame has been validated.
 *
 *   &sk_buff.csum_level indicates the number of consecutive checksums found in
 *   the packet minus one that have been verified as %CHECKSUM_UNNECESSARY.
 *   For instance if a device receives an IPv6->UDP->GRE->IPv4->TCP packet
 *   and a device is able to verify the checksums for UDP (possibly zero),
 *   GRE (checksum flag is set) and TCP, &sk_buff.csum_level would be set to
 *   two. If the device were only able to verify the UDP checksum and not
 *   GRE, either because it doesn't support GRE checksum or because GRE
 *   checksum is bad, skb->csum_level would be set to zero (TCP checksum is
 *   not considered in this case).
 *
 * - %CHECKSUM_COMPLETE
 *
 *   This is the most generic way. The device supplied checksum of the _whole_
 *   packet as seen by netif_rx() and fills in &sk_buff.csum. This means the
 *   hardware doesn't need to parse L3/L4 headers to implement this.
 *
 *   Notes:
 *
 *   - Even if device supports only some protocols, but is able to produce
 *     skb->csum, it MUST use CHECKSUM_COMPLETE, not CHECKSUM_UNNECESSARY.
 *   - CHECKSUM_COMPLETE is not applicable to SCTP and FCoE protocols.
 *
 * - %CHECKSUM_PARTIAL
 *
 *   A checksum is set up to be offloaded to a device as described in the
 *   output description for CHECKSUM_PARTIAL. This may occur on a packet
 *   received directly from another Linux OS, e.g., a virtualized Linux kernel
 *   on the same host, or it may be set in the input path in GRO or remote
 *   checksum offload. For the purposes of checksum verification, the checksum
 *   referred to by skb->csum_start + skb->csum_offset and any preceding
 *   checksums in the packet are considered verified. Any checksums in the
 *   packet that are after the checksum being offloaded are not considered to
 *   be verified.
 *
 * Checksumming on transmit for non-GSO
 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 *
 * The stack requests checksum offload in the &sk_buff.ip_summed for a packet.
 * Values are:
 *
 * - %CHECKSUM_PARTIAL
 *
 *   The driver is required to checksum the packet as seen by hard_start_xmit()
 *   from &sk_buff.csum_start up to the end, and to record/write the checksum at
 *   offset &sk_buff.csum_start + &sk_buff.csum_offset.
 *   A driver may verify that the
 *   csum_start and csum_offset values are valid values given the length and
 *   offset of the packet, but it should not attempt to validate that the
 *   checksum refers to a legitimate transport layer checksum -- it is the
 *   purview of the stack to validate that csum_start and csum_offset are set
 *   correctly.
 *
 *   When the stack requests checksum offload for a packet, the driver MUST
 *   ensure that the checksum is set correctly. A driver can either offload the
 *   checksum calculation to the device, or call skb_checksum_help (in the case
 *   that the device does not support offload for a particular checksum).
 *
 *   %NETIF_F_IP_CSUM and %NETIF_F_IPV6_CSUM are being deprecated in favor of
 *   %NETIF_F_HW_CSUM. New devices should use %NETIF_F_HW_CSUM to indicate
 *   checksum offload capability.
 *   skb_csum_hwoffload_help() can be called to resolve %CHECKSUM_PARTIAL based
 *   on network device checksumming capabilities: if a packet does not match
 *   them, skb_checksum_help() or skb_crc32c_help() (depending on the value of
 *   &sk_buff.csum_not_inet, see :ref:`crc`)
 *   is called to resolve the checksum.
 *
 * - %CHECKSUM_NONE
 *
 *   The skb was already checksummed by the protocol, or a checksum is not
 *   required.
 *
 * - %CHECKSUM_UNNECESSARY
 *
 *   This has the same meaning as CHECKSUM_NONE for checksum offload on
 *   output.
 *
 * - %CHECKSUM_COMPLETE
 *
 *   Not used in checksum output. If a driver observes a packet with this value
 *   set in skbuff, it should treat the packet as if %CHECKSUM_NONE were set.
 *
 * .. _crc:
 *
 * Non-IP checksum (CRC) offloads
 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 *
 * .. flat-table::
 *   :widths: 1 10
 *
 *   * - %NETIF_F_SCTP_CRC
 *     - This feature indicates that a device is capable of
 *         offloading the SCTP CRC in a packet. To perform this offload the stack
 *         will set csum_start and csum_offset accordingly, set ip_summed to
 *         %CHECKSUM_PARTIAL and set csum_not_inet to 1, to provide an indication
 *         in the skbuff that the %CHECKSUM_PARTIAL refers to CRC32c.
 *         A driver that supports both IP checksum offload and SCTP CRC32c offload
 *         must verify which offload is configured for a packet by testing the
 *         value of &sk_buff.csum_not_inet; skb_crc32c_csum_help() is provided to
 *         resolve %CHECKSUM_PARTIAL on skbs where csum_not_inet is set to 1.
 *
 *   * - %NETIF_F_FCOE_CRC
 *     - This feature indicates that a device is capable of offloading the FCOE
 *         CRC in a packet. To perform this offload the stack will set ip_summed
 *         to %CHECKSUM_PARTIAL and set csum_start and csum_offset
 *         accordingly. Note that there is no indication in the skbuff that the
 *         %CHECKSUM_PARTIAL refers to an FCOE checksum, so a driver that supports
 *         both IP checksum offload and FCOE CRC offload must verify which offload
 *         is configured for a packet, presumably by inspecting packet headers.
 *
 * Checksumming on output with GSO
 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 *
 * In the case of a GSO packet (skb_is_gso() is true), checksum offload
 * is implied by the SKB_GSO_* flags in gso_type. Most obviously, if the
 * gso_type is %SKB_GSO_TCPV4 or %SKB_GSO_TCPV6, TCP checksum offload as
 * part of the GSO operation is implied. If a checksum is being offloaded
 * with GSO then ip_summed is %CHECKSUM_PARTIAL, and both csum_start and
 * csum_offset are set to refer to the outermost checksum being offloaded
 * (two offloaded checksums are possible with UDP encapsulation).
 */

/* Don't change this without changing skb_csum_unnecessary! */
#define CHECKSUM_NONE                0
#define CHECKSUM_UNNECESSARY        1
#define CHECKSUM_COMPLETE        2
#define CHECKSUM_PARTIAL        3

/* Maximum value in skb->csum_level */
#define SKB_MAX_CSUM_LEVEL        3

#define SKB_DATA_ALIGN(X)        ALIGN(X, SMP_CACHE_BYTES)
#define SKB_WITH_OVERHEAD(X)        \
        ((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))

/* For X bytes available in skb->head, what is the minimal
 * allocation needed, knowing struct skb_shared_info needs
 * to be aligned.
 */
#define SKB_HEAD_ALIGN(X) (SKB_DATA_ALIGN(X) + \
        SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))

#define SKB_MAX_ORDER(X, ORDER) \
        SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X))
#define SKB_MAX_HEAD(X)                (SKB_MAX_ORDER((X), 0))
#define SKB_MAX_ALLOC                (SKB_MAX_ORDER(0, 2))

/* return minimum truesize of one skb containing X bytes of data */
#define SKB_TRUESIZE(X) ((X) +                                                \
                         SKB_DATA_ALIGN(sizeof(struct sk_buff)) +        \
                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))

struct ahash_request;
struct net_device;
struct scatterlist;
struct pipe_inode_info;
struct iov_iter;
struct napi_struct;
struct bpf_prog;
union bpf_attr;
struct skb_ext;
struct ts_config;

#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
struct nf_bridge_info {
        enum {
                BRNF_PROTO_UNCHANGED,
                BRNF_PROTO_8021Q,
                BRNF_PROTO_PPPOE
        } orig_proto:8;
        u8                        pkt_otherhost:1;
        u8                        in_prerouting:1;
        u8                        bridged_dnat:1;
        u8                        sabotage_in_done:1;
        __u16                        frag_max_size;
        int                        physinif;

        /* always valid & non-NULL from FORWARD on, for physdev match */
        struct net_device        *physoutdev;
        union {
                /* prerouting: detect dnat in orig/reply direction */
                __be32          ipv4_daddr;
                struct in6_addr ipv6_daddr;

                /* after prerouting + nat detected: store original source
                 * mac since neigh resolution overwrites it, only used while
                 * skb is out in neigh layer.
                 */
                char neigh_header[8];
        };
};
#endif

#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
/* Chain in tc_skb_ext will be used to share the tc chain with
 * ovs recirc_id. It will be set to the current chain by tc
 * and read by ovs to recirc_id.
 */
struct tc_skb_ext {
        union {
                u64 act_miss_cookie;
                __u32 chain;
        };
        __u16 mru;
        __u16 zone;
        u8 post_ct:1;
        u8 post_ct_snat:1;
        u8 post_ct_dnat:1;
        u8 act_miss:1; /* Set if act_miss_cookie is used */
        u8 l2_miss:1; /* Set by bridge upon FDB or MDB miss */
};
#endif

struct sk_buff_head {
        /* These two members must be first to match sk_buff. */
        struct_group_tagged(sk_buff_list, list,
                struct sk_buff        *next;
                struct sk_buff        *prev;
        );

        __u32                qlen;
        spinlock_t        lock;
};

struct sk_buff;

#ifndef CONFIG_MAX_SKB_FRAGS
# define CONFIG_MAX_SKB_FRAGS 17
#endif

#define MAX_SKB_FRAGS CONFIG_MAX_SKB_FRAGS

/* Set skb_shinfo(skb)->gso_size to this in case you want skb_segment to
 * segment using its current segmentation instead.
 */
#define GSO_BY_FRAGS        0xFFFF

typedef struct skb_frag {
        netmem_ref netmem;
        unsigned int len;
        unsigned int offset;
} skb_frag_t;

/**
 * skb_frag_size() - Returns the size of a skb fragment
 * @frag: skb fragment
 */
static inline unsigned int skb_frag_size(const skb_frag_t *frag)
{
        return frag->len;
}

/**
 * skb_frag_size_set() - Sets the size of a skb fragment
 * @frag: skb fragment
 * @size: size of fragment
 */
static inline void skb_frag_size_set(skb_frag_t *frag, unsigned int size)
{
        frag->len = size;
}

/**
 * skb_frag_size_add() - Increments the size of a skb fragment by @delta
 * @frag: skb fragment
 * @delta: value to add
 */
static inline void skb_frag_size_add(skb_frag_t *frag, int delta)
{
        frag->len += delta;
}

/**
 * skb_frag_size_sub() - Decrements the size of a skb fragment by @delta
 * @frag: skb fragment
 * @delta: value to subtract
 */
static inline void skb_frag_size_sub(skb_frag_t *frag, int delta)
{
        frag->len -= delta;
}

/**
 * skb_frag_must_loop - Test if %p is a high memory page
 * @p: fragment's page
 */
static inline bool skb_frag_must_loop(struct page *p)
{
#if defined(CONFIG_HIGHMEM)
        if (IS_ENABLED(CONFIG_DEBUG_KMAP_LOCAL_FORCE_MAP) || PageHighMem(p))
                return true;
#endif
        return false;
}

/**
 *        skb_frag_foreach_page - loop over pages in a fragment
 *
 *        @f:                skb frag to operate on
 *        @f_off:                offset from start of f->netmem
 *        @f_len:                length from f_off to loop over
 *        @p:                (temp var) current page
 *        @p_off:                (temp var) offset from start of current page,
 *                                   non-zero only on first page.
 *        @p_len:                (temp var) length in current page,
 *                                   < PAGE_SIZE only on first and last page.
 *        @copied:        (temp var) length so far, excluding current p_len.
 *
 *        A fragment can hold a compound page, in which case per-page
 *        operations, notably kmap_atomic, must be called for each
 *        regular page.
 */
#define skb_frag_foreach_page(f, f_off, f_len, p, p_off, p_len, copied)        \
        for (p = skb_frag_page(f) + ((f_off) >> PAGE_SHIFT),                \
             p_off = (f_off) & (PAGE_SIZE - 1),                                \
             p_len = skb_frag_must_loop(p) ?                                \
             min_t(u32, f_len, PAGE_SIZE - p_off) : f_len,                \
             copied = 0;                                                \
             copied < f_len;                                                \
             copied += p_len, p++, p_off = 0,                                \
             p_len = min_t(u32, f_len - copied, PAGE_SIZE))                \

/**
 * struct skb_shared_hwtstamps - hardware time stamps
 * @hwtstamp:                hardware time stamp transformed into duration
 *                        since arbitrary point in time
 * @netdev_data:        address/cookie of network device driver used as
 *                        reference to actual hardware time stamp
 *
 * Software time stamps generated by ktime_get_real() are stored in
 * skb->tstamp.
 *
 * hwtstamps can only be compared against other hwtstamps from
 * the same device.
 *
 * This structure is attached to packets as part of the
 * &skb_shared_info. Use skb_hwtstamps() to get a pointer.
 */
struct skb_shared_hwtstamps {
        union {
                ktime_t        hwtstamp;
                void *netdev_data;
        };
};

/* Definitions for tx_flags in struct skb_shared_info */
enum {
        /* generate hardware time stamp */
        SKBTX_HW_TSTAMP_NOBPF = 1 << 0,

        /* generate software time stamp when queueing packet to NIC */
        SKBTX_SW_TSTAMP = 1 << 1,

        /* device driver is going to provide hardware time stamp */
        SKBTX_IN_PROGRESS = 1 << 2,

        /* generate software time stamp on packet tx completion */
        SKBTX_COMPLETION_TSTAMP = 1 << 3,

        /* generate wifi status information (where possible) */
        SKBTX_WIFI_STATUS = 1 << 4,

        /* determine hardware time stamp based on time or cycles */
        SKBTX_HW_TSTAMP_NETDEV = 1 << 5,

        /* generate software time stamp when entering packet scheduling */
        SKBTX_SCHED_TSTAMP = 1 << 6,

        /* used for bpf extension when a bpf program is loaded */
        SKBTX_BPF = 1 << 7,
};

#define SKBTX_HW_TSTAMP                (SKBTX_HW_TSTAMP_NOBPF | SKBTX_BPF)

#define SKBTX_ANY_SW_TSTAMP        (SKBTX_SW_TSTAMP    | \
                                 SKBTX_SCHED_TSTAMP | \
                                 SKBTX_BPF          | \
                                 SKBTX_COMPLETION_TSTAMP)
#define SKBTX_ANY_TSTAMP        (SKBTX_HW_TSTAMP | \
                                 SKBTX_ANY_SW_TSTAMP)

/* Definitions for flags in struct skb_shared_info */
enum {
        /* use zcopy routines */
        SKBFL_ZEROCOPY_ENABLE = BIT(0),

        /* This indicates at least one fragment might be overwritten
         * (as in vmsplice(), sendfile() ...)
         * If we need to compute a TX checksum, we'll need to copy
         * all frags to avoid possible bad checksum
         */
        SKBFL_SHARED_FRAG = BIT(1),

        /* segment contains only zerocopy data and should not be
         * charged to the kernel memory.
         */
        SKBFL_PURE_ZEROCOPY = BIT(2),

        SKBFL_DONT_ORPHAN = BIT(3),

        /* page references are managed by the ubuf_info, so it's safe to
         * use frags only up until ubuf_info is released
         */
        SKBFL_MANAGED_FRAG_REFS = BIT(4),
};

#define SKBFL_ZEROCOPY_FRAG        (SKBFL_ZEROCOPY_ENABLE | SKBFL_SHARED_FRAG)
#define SKBFL_ALL_ZEROCOPY        (SKBFL_ZEROCOPY_FRAG | SKBFL_PURE_ZEROCOPY | \
                                 SKBFL_DONT_ORPHAN | SKBFL_MANAGED_FRAG_REFS)

struct ubuf_info_ops {
        void (*complete)(struct sk_buff *, struct ubuf_info *,
                         bool zerocopy_success);
        /* has to be compatible with skb_zcopy_set() */
        int (*link_skb)(struct sk_buff *skb, struct ubuf_info *uarg);
};

/*
 * The callback notifies userspace to release buffers when skb DMA is done in
 * lower device, the skb last reference should be 0 when calling this.
 * The zerocopy_success argument is true if zero copy transmit occurred,
 * false on data copy or out of memory error caused by data copy attempt.
 * The ctx field is used to track device context.
 * The desc field is used to track userspace buffer index.
 */
struct ubuf_info {
        const struct ubuf_info_ops *ops;
        refcount_t refcnt;
        u8 flags;
};

struct ubuf_info_msgzc {
        struct ubuf_info ubuf;

        union {
                struct {
                        unsigned long desc;
                        void *ctx;
                };
                struct {
                        u32 id;
                        u16 len;
                        u16 zerocopy:1;
                        u32 bytelen;
                };
        };

        struct mmpin {
                struct user_struct *user;
                unsigned int num_pg;
        } mmp;
};

#define skb_uarg(SKB)        ((struct ubuf_info *)(skb_shinfo(SKB)->destructor_arg))
#define uarg_to_msgzc(ubuf_ptr)        container_of((ubuf_ptr), struct ubuf_info_msgzc, \
                                             ubuf)

int mm_account_pinned_pages(struct mmpin *mmp, size_t size);
void mm_unaccount_pinned_pages(struct mmpin *mmp);

/* Preserve some data across TX submission and completion.
 *
 * Note, this state is stored in the driver. Extending the layout
 * might need some special care.
 */
struct xsk_tx_metadata_compl {
        __u64 *tx_timestamp;
};

/* This data is invariant across clones and lives at
 * the end of the header data, ie. at skb->end.
 */
struct skb_shared_info {
        __u8                flags;
        __u8                meta_len;
        __u8                nr_frags;
        __u8                tx_flags;
        unsigned short        gso_size;
        /* Warning: this field is not always filled in (UFO)! */
        unsigned short        gso_segs;
        struct sk_buff        *frag_list;
        union {
                struct skb_shared_hwtstamps hwtstamps;
                struct xsk_tx_metadata_compl xsk_meta;
        };
        unsigned int        gso_type;
        u32                tskey;

        /*
         * Warning : all fields before dataref are cleared in __alloc_skb()
         */
        atomic_t        dataref;

        union {
                struct {
                        u32                xdp_frags_size;
                        u32                xdp_frags_truesize;
                };

                /*
                 * Intermediate layers must ensure that destructor_arg
                 * remains valid until skb destructor.
                 */
                void                *destructor_arg;
        };

        /* must be last field, see pskb_expand_head() */
        skb_frag_t        frags[MAX_SKB_FRAGS];
};

/**
 * DOC: dataref and headerless skbs
 *
 * Transport layers send out clones of payload skbs they hold for
 * retransmissions. To allow lower layers of the stack to prepend their headers
 * we split &skb_shared_info.dataref into two halves.
 * The lower 16 bits count the overall number of references.
 * The higher 16 bits indicate how many of the references are payload-only.
 * skb_header_cloned() checks if skb is allowed to add / write the headers.
 *
 * The creator of the skb (e.g. TCP) marks its skb as &sk_buff.nohdr
 * (via __skb_header_release()). Any clone created from marked skb will get
 * &sk_buff.hdr_len populated with the available headroom.
 * If there's the only clone in existence it's able to modify the headroom
 * at will. The sequence of calls inside the transport layer is::
 *
 *  <alloc skb>
 *  skb_reserve()
 *  __skb_header_release()
 *  skb_clone()
 *  // send the clone down the stack
 *
 * This is not a very generic construct and it depends on the transport layers
 * doing the right thing. In practice there's usually only one payload-only skb.
 * Having multiple payload-only skbs with different lengths of hdr_len is not
 * possible. The payload-only skbs should never leave their owner.
 */
#define SKB_DATAREF_SHIFT 16
#define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1)


enum {
        SKB_FCLONE_UNAVAILABLE,        /* skb has no fclone (from head_cache) */
        SKB_FCLONE_ORIG,        /* orig skb (from fclone_cache) */
        SKB_FCLONE_CLONE,        /* companion fclone skb (from fclone_cache) */
};

enum {
        SKB_GSO_TCPV4 = 1 << 0,

        /* This indicates the skb is from an untrusted source. */
        SKB_GSO_DODGY = 1 << 1,

        /* This indicates the tcp segment has CWR set. */
        SKB_GSO_TCP_ECN = 1 << 2,

        SKB_GSO_TCP_FIXEDID = 1 << 3,

        SKB_GSO_TCPV6 = 1 << 4,

        SKB_GSO_FCOE = 1 << 5,

        SKB_GSO_GRE = 1 << 6,

        SKB_GSO_GRE_CSUM = 1 << 7,

        SKB_GSO_IPXIP4 = 1 << 8,

        SKB_GSO_IPXIP6 = 1 << 9,

        SKB_GSO_UDP_TUNNEL = 1 << 10,

        SKB_GSO_UDP_TUNNEL_CSUM = 1 << 11,

        SKB_GSO_PARTIAL = 1 << 12,

        SKB_GSO_TUNNEL_REMCSUM = 1 << 13,

        SKB_GSO_SCTP = 1 << 14,

        SKB_GSO_ESP = 1 << 15,

        SKB_GSO_UDP = 1 << 16,

        SKB_GSO_UDP_L4 = 1 << 17,

        SKB_GSO_FRAGLIST = 1 << 18,

        SKB_GSO_TCP_ACCECN = 1 << 19,
};

#if BITS_PER_LONG > 32
#define NET_SKBUFF_DATA_USES_OFFSET 1
#endif

#ifdef NET_SKBUFF_DATA_USES_OFFSET
typedef unsigned int sk_buff_data_t;
#else
typedef unsigned char *sk_buff_data_t;
#endif

enum skb_tstamp_type {
        SKB_CLOCK_REALTIME,
        SKB_CLOCK_MONOTONIC,
        SKB_CLOCK_TAI,
        __SKB_CLOCK_MAX = SKB_CLOCK_TAI,
};

/**
 * DOC: Basic sk_buff geometry
 *
 * struct sk_buff itself is a metadata structure and does not hold any packet
 * data. All the data is held in associated buffers.
 *
 * &sk_buff.head points to the main "head" buffer. The head buffer is divided
 * into two parts:
 *
 *  - data buffer, containing headers and sometimes payload;
 *    this is the part of the skb operated on by the common helpers
 *    such as skb_put() or skb_pull();
 *  - shared info (struct skb_shared_info) which holds an array of pointers
 *    to read-only data in the (page, offset, length) format.
 *
 * Optionally &skb_shared_info.frag_list may point to another skb.
 *
 * Basic diagram may look like this::
 *
 *                                  ---------------
 *                                 | sk_buff       |
 *                                  ---------------
 *     ,---------------------------  + head
 *    /          ,-----------------  + data
 *   /          /      ,-----------  + tail
 *  |          |      |            , + end
 *  |          |      |           |
 *  v          v      v           v
 *   -----------------------------------------------
 *  | headroom | data |  tailroom | skb_shared_info |
 *   -----------------------------------------------
 *                                 + [page frag]
 *                                 + [page frag]
 *                                 + [page frag]
 *                                 + [page frag]       ---------
 *                                 + frag_list    --> | sk_buff |
 *                                                     ---------
 *
 */

/**
 *        struct sk_buff - socket buffer
 *        @next: Next buffer in list
 *        @prev: Previous buffer in list
 *        @tstamp: Time we arrived/left
 *        @skb_mstamp_ns: (aka @tstamp) earliest departure time; start point
 *                for retransmit timer
 *        @rbnode: RB tree node, alternative to next/prev for netem/tcp
 *        @list: queue head
 *        @ll_node: anchor in an llist (eg socket defer_list)
 *        @sk: Socket we are owned by
 *        @dev: Device we arrived on/are leaving by
 *        @dev_scratch: (aka @dev) alternate use of @dev when @dev would be %NULL
 *        @cb: Control buffer. Free for use by every layer. Put private vars here
 *        @_skb_refdst: destination entry (with norefcount bit)
 *        @len: Length of actual data
 *        @data_len: Data length
 *        @mac_len: Length of link layer header
 *        @hdr_len: writable header length of cloned skb
 *        @csum: Checksum (must include start/offset pair)
 *        @csum_start: Offset from skb->head where checksumming should start
 *        @csum_offset: Offset from csum_start where checksum should be stored
 *        @priority: Packet queueing priority
 *        @ignore_df: allow local fragmentation
 *        @cloned: Head may be cloned (check refcnt to be sure)
 *        @ip_summed: Driver fed us an IP checksum
 *        @nohdr: Payload reference only, must not modify header
 *        @pkt_type: Packet class
 *        @fclone: skbuff clone status
 *        @ipvs_property: skbuff is owned by ipvs
 *        @inner_protocol_type: whether the inner protocol is
 *                ENCAP_TYPE_ETHER or ENCAP_TYPE_IPPROTO
 *        @remcsum_offload: remote checksum offload is enabled
 *        @offload_fwd_mark: Packet was L2-forwarded in hardware
 *        @offload_l3_fwd_mark: Packet was L3-forwarded in hardware
 *        @tc_skip_classify: do not classify packet. set by IFB device
 *        @tc_at_ingress: used within tc_classify to distinguish in/egress
 *        @redirected: packet was redirected by packet classifier
 *        @from_ingress: packet was redirected from the ingress path
 *        @nf_skip_egress: packet shall skip nf egress - see netfilter_netdev.h
 *        @peeked: this packet has been seen already, so stats have been
 *                done for it, don't do them again
 *        @nf_trace: netfilter packet trace flag
 *        @protocol: Packet protocol from driver
 *        @destructor: Destruct function
 *        @tcp_tsorted_anchor: list structure for TCP (tp->tsorted_sent_queue)
 *        @_sk_redir: socket redirection information for skmsg
 *        @_nfct: Associated connection, if any (with nfctinfo bits)
 *        @skb_iif: ifindex of device we arrived on
 *        @tc_index: Traffic control index
 *        @hash: the packet hash
 *        @queue_mapping: Queue mapping for multiqueue devices
 *        @head_frag: skb was allocated from page fragments,
 *                not allocated by kmalloc() or vmalloc().
 *        @pfmemalloc: skbuff was allocated from PFMEMALLOC reserves
 *        @pp_recycle: mark the packet for recycling instead of freeing (implies
 *                page_pool support on driver)
 *        @active_extensions: active extensions (skb_ext_id types)
 *        @ndisc_nodetype: router type (from link layer)
 *        @ooo_okay: allow the mapping of a socket to a queue to be changed
 *        @l4_hash: indicate hash is a canonical 4-tuple hash over transport
 *                ports.
 *        @sw_hash: indicates hash was computed in software stack
 *        @wifi_acked_valid: wifi_acked was set
 *        @wifi_acked: whether frame was acked on wifi or not
 *        @no_fcs:  Request NIC to treat last 4 bytes as Ethernet FCS
 *        @encapsulation: indicates the inner headers in the skbuff are valid
 *        @encap_hdr_csum: software checksum is needed
 *        @csum_valid: checksum is already valid
 *        @csum_not_inet: use CRC32c to resolve CHECKSUM_PARTIAL
 *        @csum_complete_sw: checksum was completed by software
 *        @csum_level: indicates the number of consecutive checksums found in
 *                the packet minus one that have been verified as
 *                CHECKSUM_UNNECESSARY (max 3)
 *        @unreadable: indicates that at least 1 of the fragments in this skb is
 *                unreadable.
 *        @dst_pending_confirm: need to confirm neighbour
 *        @decrypted: Decrypted SKB
 *        @slow_gro: state present at GRO time, slower prepare step required
 *        @tstamp_type: When set, skb->tstamp has the
 *                delivery_time clock base of skb->tstamp.
 *        @napi_id: id of the NAPI struct this skb came from
 *        @sender_cpu: (aka @napi_id) source CPU in XPS
 *        @alloc_cpu: CPU which did the skb allocation.
 *        @secmark: security marking
 *        @mark: Generic packet mark
 *        @reserved_tailroom: (aka @mark) number of bytes of free space available
 *                at the tail of an sk_buff
 *        @vlan_all: vlan fields (proto & tci)
 *        @vlan_proto: vlan encapsulation protocol
 *        @vlan_tci: vlan tag control information
 *        @inner_protocol: Protocol (encapsulation)
 *        @inner_ipproto: (aka @inner_protocol) stores ipproto when
 *                skb->inner_protocol_type == ENCAP_TYPE_IPPROTO;
 *        @inner_transport_header: Inner transport layer header (encapsulation)
 *        @inner_network_header: Network layer header (encapsulation)
 *        @inner_mac_header: Link layer header (encapsulation)
 *        @transport_header: Transport layer header
 *        @network_header: Network layer header
 *        @mac_header: Link layer header
 *        @kcov_handle: KCOV remote handle for remote coverage collection
 *        @tail: Tail pointer
 *        @end: End pointer
 *        @head: Head of buffer
 *        @data: Data head pointer
 *        @truesize: Buffer size
 *        @users: User count - see {datagram,tcp}.c
 *        @extensions: allocated extensions, valid if active_extensions is nonzero
 */

struct sk_buff {
        union {
                struct {
                        /* These two members must be first to match sk_buff_head. */
                        struct sk_buff                *next;
                        struct sk_buff                *prev;

                        union {
                                struct net_device        *dev;
                                /* Some protocols might use this space to store information,
                                 * while device pointer would be NULL.
                                 * UDP receive path is one user.
                                 */
                                unsigned long                dev_scratch;
                        };
                };
                struct rb_node                rbnode; /* used in netem, ip4 defrag, and tcp stack */
                struct list_head        list;
                struct llist_node        ll_node;
        };

        struct sock                *sk;

        union {
                ktime_t                tstamp;
                u64                skb_mstamp_ns; /* earliest departure time */
        };
        /*
         * This is the control buffer. It is free to use for every
         * layer. Please put your private variables there. If you
         * want to keep them across layers you have to do a skb_clone()
         * first. This is owned by whoever has the skb queued ATM.
         */
        char                        cb[48] __aligned(8);

        union {
                struct {
                        unsigned long        _skb_refdst;
                        void                (*destructor)(struct sk_buff *skb);
                };
                struct list_head        tcp_tsorted_anchor;
#ifdef CONFIG_NET_SOCK_MSG
                unsigned long                _sk_redir;
#endif
        };

#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
        unsigned long                 _nfct;
#endif
        unsigned int                len,
                                data_len;
        __u16                        mac_len,
                                hdr_len;

        /* Following fields are _not_ copied in __copy_skb_header()
         * Note that queue_mapping is here mostly to fill a hole.
         */
        __u16                        queue_mapping;

/* if you move cloned around you also must adapt those constants */
#ifdef __BIG_ENDIAN_BITFIELD
#define CLONED_MASK        (1 << 7)
#else
#define CLONED_MASK        1
#endif
#define CLONED_OFFSET                offsetof(struct sk_buff, __cloned_offset)

        /* private: */
        __u8                        __cloned_offset[0];
        /* public: */
        __u8                        cloned:1,
                                nohdr:1,
                                fclone:2,
                                peeked:1,
                                head_frag:1,
                                pfmemalloc:1,
                                pp_recycle:1; /* page_pool recycle indicator */
#ifdef CONFIG_SKB_EXTENSIONS
        __u8                        active_extensions;
#endif

        /* Fields enclosed in headers group are copied
         * using a single memcpy() in __copy_skb_header()
         */
        struct_group(headers,

        /* private: */
        __u8                        __pkt_type_offset[0];
        /* public: */
        __u8                        pkt_type:3; /* see PKT_TYPE_MAX */
        __u8                        ignore_df:1;
        __u8                        dst_pending_confirm:1;
        __u8                        ip_summed:2;
        __u8                        ooo_okay:1;

        /* private: */
        __u8                        __mono_tc_offset[0];
        /* public: */
        __u8                        tstamp_type:2;        /* See skb_tstamp_type */
#ifdef CONFIG_NET_XGRESS
        __u8                        tc_at_ingress:1;        /* See TC_AT_INGRESS_MASK */
        __u8                        tc_skip_classify:1;
#endif
        __u8                        remcsum_offload:1;
        __u8                        csum_complete_sw:1;
        __u8                        csum_level:2;
        __u8                        inner_protocol_type:1;

        __u8                        l4_hash:1;
        __u8                        sw_hash:1;
#ifdef CONFIG_WIRELESS
        __u8                        wifi_acked_valid:1;
        __u8                        wifi_acked:1;
#endif
        __u8                        no_fcs:1;
        /* Indicates the inner headers are valid in the skbuff. */
        __u8                        encapsulation:1;
        __u8                        encap_hdr_csum:1;
        __u8                        csum_valid:1;
#ifdef CONFIG_IPV6_NDISC_NODETYPE
        __u8                        ndisc_nodetype:2;
#endif

#if IS_ENABLED(CONFIG_IP_VS)
        __u8                        ipvs_property:1;
#endif
#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || IS_ENABLED(CONFIG_NF_TABLES)
        __u8                        nf_trace:1;
#endif
#ifdef CONFIG_NET_SWITCHDEV
        __u8                        offload_fwd_mark:1;
        __u8                        offload_l3_fwd_mark:1;
#endif
        __u8                        redirected:1;
#ifdef CONFIG_NET_REDIRECT
        __u8                        from_ingress:1;
#endif
#ifdef CONFIG_NETFILTER_SKIP_EGRESS
        __u8                        nf_skip_egress:1;
#endif
#ifdef CONFIG_SKB_DECRYPTED
        __u8                        decrypted:1;
#endif
        __u8                        slow_gro:1;
#if IS_ENABLED(CONFIG_IP_SCTP)
        __u8                        csum_not_inet:1;
#endif
        __u8                        unreadable:1;
#if defined(CONFIG_NET_SCHED) || defined(CONFIG_NET_XGRESS)
        __u16                        tc_index;        /* traffic control index */
#endif

        u16                        alloc_cpu;

        union {
                __wsum                csum;
                struct {
                        __u16        csum_start;
                        __u16        csum_offset;
                };
        };
        __u32                        priority;
        int                        skb_iif;
        __u32                        hash;
        union {
                u32                vlan_all;
                struct {
                        __be16        vlan_proto;
                        __u16        vlan_tci;
                };
        };
#if defined(CONFIG_NET_RX_BUSY_POLL) || defined(CONFIG_XPS)
        union {
                unsigned int        napi_id;
                unsigned int        sender_cpu;
        };
#endif
#ifdef CONFIG_NETWORK_SECMARK
        __u32                secmark;
#endif

        union {
                __u32                mark;
                __u32                reserved_tailroom;
        };

        union {
                __be16                inner_protocol;
                __u8                inner_ipproto;
        };

        __u16                        inner_transport_header;
        __u16                        inner_network_header;
        __u16                        inner_mac_header;

        __be16                        protocol;
        __u16                        transport_header;
        __u16                        network_header;
        __u16                        mac_header;

#ifdef CONFIG_KCOV
        u64                        kcov_handle;
#endif

        ); /* end headers group */

        /* These elements must be at the end, see alloc_skb() for details.  */
        sk_buff_data_t                tail;
        sk_buff_data_t                end;
        unsigned char                *head,
                                *data;
        unsigned int                truesize;
        refcount_t                users;

#ifdef CONFIG_SKB_EXTENSIONS
        /* only usable after checking ->active_extensions != 0 */
        struct skb_ext                *extensions;
#endif
};

/* if you move pkt_type around you also must adapt those constants */
#ifdef __BIG_ENDIAN_BITFIELD
#define PKT_TYPE_MAX        (7 << 5)
#else
#define PKT_TYPE_MAX        7
#endif
#define PKT_TYPE_OFFSET                offsetof(struct sk_buff, __pkt_type_offset)

/* if you move tc_at_ingress or tstamp_type
 * around, you also must adapt these constants.
 */
#ifdef __BIG_ENDIAN_BITFIELD
#define SKB_TSTAMP_TYPE_MASK                (3 << 6)
#define SKB_TSTAMP_TYPE_RSHIFT                (6)
#define TC_AT_INGRESS_MASK                (1 << 5)
#else
#define SKB_TSTAMP_TYPE_MASK                (3)
#define TC_AT_INGRESS_MASK                (1 << 2)
#endif
#define SKB_BF_MONO_TC_OFFSET                offsetof(struct sk_buff, __mono_tc_offset)

#ifdef __KERNEL__
/*
 *        Handling routines are only of interest to the kernel
 */

#define SKB_ALLOC_FCLONE        0x01
#define SKB_ALLOC_RX                0x02
#define SKB_ALLOC_NAPI                0x04

/**
 * skb_pfmemalloc - Test if the skb was allocated from PFMEMALLOC reserves
 * @skb: buffer
 */
static inline bool skb_pfmemalloc(const struct sk_buff *skb)
{
        return unlikely(skb->pfmemalloc);
}

/*
 * skb might have a dst pointer attached, refcounted or not.
 * _skb_refdst low order bit is set if refcount was _not_ taken
 */
#define SKB_DST_NOREF        1UL
#define SKB_DST_PTRMASK        ~(SKB_DST_NOREF)

/**
 * skb_dst - returns skb dst_entry
 * @skb: buffer
 *
 * Returns: skb dst_entry, regardless of reference taken or not.
 */
static inline struct dst_entry *skb_dst(const struct sk_buff *skb)
{
        /* If refdst was not refcounted, check we still are in a
         * rcu_read_lock section
         */
        WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) &&
                !rcu_read_lock_held() &&
                !rcu_read_lock_bh_held());
        return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK);
}

/**
 * skb_dst_set - sets skb dst
 * @skb: buffer
 * @dst: dst entry
 *
 * Sets skb dst, assuming a reference was taken on dst and should
 * be released by skb_dst_drop()
 */
static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst)
{
        skb->slow_gro |= !!dst;
        skb->_skb_refdst = (unsigned long)dst;
}

/**
 * skb_dst_set_noref - sets skb dst, hopefully, without taking reference
 * @skb: buffer
 * @dst: dst entry
 *
 * Sets skb dst, assuming a reference was not taken on dst.
 * If dst entry is cached, we do not take reference and dst_release
 * will be avoided by refdst_drop. If dst entry is not cached, we take
 * reference, so that last dst_release can destroy the dst immediately.
 */
static inline void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst)
{
        WARN_ON(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
        skb->slow_gro |= !!dst;
        skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF;
}

/**
 * skb_dst_is_noref - Test if skb dst isn't refcounted
 * @skb: buffer
 */
static inline bool skb_dst_is_noref(const struct sk_buff *skb)
{
        return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb);
}

/* For mangling skb->pkt_type from user space side from applications
 * such as nft, tc, etc, we only allow a conservative subset of
 * possible pkt_types to be set.
*/
static inline bool skb_pkt_type_ok(u32 ptype)
{
        return ptype <= PACKET_OTHERHOST;
}

/**
 * skb_napi_id - Returns the skb's NAPI id
 * @skb: buffer
 */
static inline unsigned int skb_napi_id(const struct sk_buff *skb)
{
#ifdef CONFIG_NET_RX_BUSY_POLL
        return skb->napi_id;
#else
        return 0;
#endif
}

static inline bool skb_wifi_acked_valid(const struct sk_buff *skb)
{
#ifdef CONFIG_WIRELESS
        return skb->wifi_acked_valid;
#else
        return 0;
#endif
}

/**
 * skb_unref - decrement the skb's reference count
 * @skb: buffer
 *
 * Returns: true if we can free the skb.
 */
static inline bool skb_unref(struct sk_buff *skb)
{
        if (unlikely(!skb))
                return false;
        if (!IS_ENABLED(CONFIG_DEBUG_NET) && likely(refcount_read(&skb->users) == 1))
                smp_rmb();
        else if (likely(!refcount_dec_and_test(&skb->users)))
                return false;

        return true;
}

static inline bool skb_data_unref(const struct sk_buff *skb,
                                  struct skb_shared_info *shinfo)
{
        int bias;

        if (!skb->cloned)
                return true;

        bias = skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1;

        if (atomic_read(&shinfo->dataref) == bias)
                smp_rmb();
        else if (atomic_sub_return(bias, &shinfo->dataref))
                return false;

        return true;
}

void __fix_address sk_skb_reason_drop(struct sock *sk, struct sk_buff *skb,
                                      enum skb_drop_reason reason);

static inline void
kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason)
{
        sk_skb_reason_drop(NULL, skb, reason);
}

/**
 *        kfree_skb - free an sk_buff with 'NOT_SPECIFIED' reason
 *        @skb: buffer to free
 */
static inline void kfree_skb(struct sk_buff *skb)
{
        kfree_skb_reason(skb, SKB_DROP_REASON_NOT_SPECIFIED);
}

void skb_release_head_state(struct sk_buff *skb);
void kfree_skb_list_reason(struct sk_buff *segs,
                           enum skb_drop_reason reason);
void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt);
void skb_tx_error(struct sk_buff *skb);

static inline void kfree_skb_list(struct sk_buff *segs)
{
        kfree_skb_list_reason(segs, SKB_DROP_REASON_NOT_SPECIFIED);
}

#ifdef CONFIG_TRACEPOINTS
void consume_skb(struct sk_buff *skb);
#else
static inline void consume_skb(struct sk_buff *skb)
{
        return kfree_skb(skb);
}
#endif

void __consume_stateless_skb(struct sk_buff *skb);
void  __kfree_skb(struct sk_buff *skb);

void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
                      bool *fragstolen, int *delta_truesize);

struct sk_buff *__alloc_skb(unsigned int size, gfp_t priority, int flags,
                            int node);
struct sk_buff *__build_skb(void *data, unsigned int frag_size);
struct sk_buff *build_skb(void *data, unsigned int frag_size);
struct sk_buff *build_skb_around(struct sk_buff *skb,
                                 void *data, unsigned int frag_size);
void skb_attempt_defer_free(struct sk_buff *skb);

u32 napi_skb_cache_get_bulk(void **skbs, u32 n);
struct sk_buff *napi_build_skb(void *data, unsigned int frag_size);
struct sk_buff *slab_build_skb(void *data);

/**
 * alloc_skb - allocate a network buffer
 * @size: size to allocate
 * @priority: allocation mask
 *
 * This function is a convenient wrapper around __alloc_skb().
 */
static inline struct sk_buff *alloc_skb(unsigned int size,
                                        gfp_t priority)
{
        return __alloc_skb(size, priority, 0, NUMA_NO_NODE);
}

struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
                                     unsigned long data_len,
                                     int max_page_order,
                                     int *errcode,
                                     gfp_t gfp_mask);
struct sk_buff *alloc_skb_for_msg(struct sk_buff *first);

/* Layout of fast clones : [skb1][skb2][fclone_ref] */
struct sk_buff_fclones {
        struct sk_buff        skb1;

        struct sk_buff        skb2;

        refcount_t        fclone_ref;
};

/**
 *        skb_fclone_busy - check if fclone is busy
 *        @sk: socket
 *        @skb: buffer
 *
 * Returns: true if skb is a fast clone, and its clone is not freed.
 * Some drivers call skb_orphan() in their ndo_start_xmit(),
 * so we also check that didn't happen.
 */
static inline bool skb_fclone_busy(const struct sock *sk,
                                   const struct sk_buff *skb)
{
        const struct sk_buff_fclones *fclones;

        fclones = container_of(skb, struct sk_buff_fclones, skb1);

        return skb->fclone == SKB_FCLONE_ORIG &&
               refcount_read(&fclones->fclone_ref) > 1 &&
               READ_ONCE(fclones->skb2.sk) == sk;
}

/**
 * alloc_skb_fclone - allocate a network buffer from fclone cache
 * @size: size to allocate
 * @priority: allocation mask
 *
 * This function is a convenient wrapper around __alloc_skb().
 */
static inline struct sk_buff *alloc_skb_fclone(unsigned int size,
                                               gfp_t priority)
{
        return __alloc_skb(size, priority, SKB_ALLOC_FCLONE, NUMA_NO_NODE);
}

struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src);
void skb_headers_offset_update(struct sk_buff *skb, int off);
int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask);
struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t priority);
void skb_copy_header(struct sk_buff *new, const struct sk_buff *old);
struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t priority);
struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
                                   gfp_t gfp_mask, bool fclone);
static inline struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom,
                                          gfp_t gfp_mask)
{
        return __pskb_copy_fclone(skb, headroom, gfp_mask, false);
}

int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, gfp_t gfp_mask);
struct sk_buff *skb_realloc_headroom(struct sk_buff *skb,
                                     unsigned int headroom);
struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom);
struct sk_buff *skb_copy_expand(const struct sk_buff *skb, int newheadroom,
                                int newtailroom, gfp_t priority);
int __must_check skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
                                     int offset, int len);
int __must_check skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg,
                              int offset, int len);
int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer);
int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error);

/**
 *        skb_pad                        -        zero pad the tail of an skb
 *        @skb: buffer to pad
 *        @pad: space to pad
 *
 *        Ensure that a buffer is followed by a padding area that is zero
 *        filled. Used by network drivers which may DMA or transfer data
 *        beyond the buffer end onto the wire.
 *
 *        May return error in out of memory cases. The skb is freed on error.
 */
static inline int skb_pad(struct sk_buff *skb, int pad)
{
        return __skb_pad(skb, pad, true);
}
#define dev_kfree_skb(a)        consume_skb(a)

int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
                         int offset, size_t size, size_t max_frags);

struct skb_seq_state {
        __u32                lower_offset;
        __u32                upper_offset;
        __u32                frag_idx;
        __u32                stepped_offset;
        struct sk_buff        *root_skb;
        struct sk_buff        *cur_skb;
        __u8                *frag_data;
        __u32                frag_off;
};

void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
                          unsigned int to, struct skb_seq_state *st);
unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
                          struct skb_seq_state *st);
void skb_abort_seq_read(struct skb_seq_state *st);
int skb_copy_seq_read(struct skb_seq_state *st, int offset, void *to, int len);

unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
                           unsigned int to, struct ts_config *config);

/*
 * Packet hash types specify the type of hash in skb_set_hash.
 *
 * Hash types refer to the protocol layer addresses which are used to
 * construct a packet's hash. The hashes are used to differentiate or identify
 * flows of the protocol layer for the hash type. Hash types are either
 * layer-2 (L2), layer-3 (L3), or layer-4 (L4).
 *
 * Properties of hashes:
 *
 * 1) Two packets in different flows have different hash values
 * 2) Two packets in the same flow should have the same hash value
 *
 * A hash at a higher layer is considered to be more specific. A driver should
 * set the most specific hash possible.
 *
 * A driver cannot indicate a more specific hash than the layer at which a hash
 * was computed. For instance an L3 hash cannot be set as an L4 hash.
 *
 * A driver may indicate a hash level which is less specific than the
 * actual layer the hash was computed on. For instance, a hash computed
 * at L4 may be considered an L3 hash. This should only be done if the
 * driver can't unambiguously determine that the HW computed the hash at
 * the higher layer. Note that the "should" in the second property above
 * permits this.
 */
enum pkt_hash_types {
        PKT_HASH_TYPE_NONE,        /* Undefined type */
        PKT_HASH_TYPE_L2,        /* Input: src_MAC, dest_MAC */
        PKT_HASH_TYPE_L3,        /* Input: src_IP, dst_IP */
        PKT_HASH_TYPE_L4,        /* Input: src_IP, dst_IP, src_port, dst_port */
};

static inline void skb_clear_hash(struct sk_buff *skb)
{
        skb->hash = 0;
        skb->sw_hash = 0;
        skb->l4_hash = 0;
}

static inline void skb_clear_hash_if_not_l4(struct sk_buff *skb)
{
        if (!skb->l4_hash)
                skb_clear_hash(skb);
}

static inline void
__skb_set_hash(struct sk_buff *skb, __u32 hash, bool is_sw, bool is_l4)
{
        skb->l4_hash = is_l4;
        skb->sw_hash = is_sw;
        skb->hash = hash;
}

static inline void
skb_set_hash(struct sk_buff *skb, __u32 hash, enum pkt_hash_types type)
{
        /* Used by drivers to set hash from HW */
        __skb_set_hash(skb, hash, false, type == PKT_HASH_TYPE_L4);
}

static inline void
__skb_set_sw_hash(struct sk_buff *skb, __u32 hash, bool is_l4)
{
        __skb_set_hash(skb, hash, true, is_l4);
}

u32 __skb_get_hash_symmetric_net(const struct net *net, const struct sk_buff *skb);

static inline u32 __skb_get_hash_symmetric(const struct sk_buff *skb)
{
        return __skb_get_hash_symmetric_net(NULL, skb);
}

void __skb_get_hash_net(const struct net *net, struct sk_buff *skb);
u32 skb_get_poff(const struct sk_buff *skb);
u32 __skb_get_poff(const struct sk_buff *skb, const void *data,
                   const struct flow_keys_basic *keys, int hlen);
__be32 skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
                          const void *data, int hlen_proto);

void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
                             const struct flow_dissector_key *key,
                             unsigned int key_count);

struct bpf_flow_dissector;
u32 bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
                     __be16 proto, int nhoff, int hlen, unsigned int flags);

bool __skb_flow_dissect(const struct net *net,
                        const struct sk_buff *skb,
                        struct flow_dissector *flow_dissector,
                        void *target_container, const void *data,
                        __be16 proto, int nhoff, int hlen, unsigned int flags);

static inline bool skb_flow_dissect(const struct sk_buff *skb,
                                    struct flow_dissector *flow_dissector,
                                    void *target_container, unsigned int flags)
{
        return __skb_flow_dissect(NULL, skb, flow_dissector,
                                  target_container, NULL, 0, 0, 0, flags);
}

static inline bool skb_flow_dissect_flow_keys(const struct sk_buff *skb,
                                              struct flow_keys *flow,
                                              unsigned int flags)
{
        memset(flow, 0, sizeof(*flow));
        return __skb_flow_dissect(NULL, skb, &flow_keys_dissector,
                                  flow, NULL, 0, 0, 0, flags);
}

static inline bool
skb_flow_dissect_flow_keys_basic(const struct net *net,
                                 const struct sk_buff *skb,
                                 struct flow_keys_basic *flow,
                                 const void *data, __be16 proto,
                                 int nhoff, int hlen, unsigned int flags)
{
        memset(flow, 0, sizeof(*flow));
        return __skb_flow_dissect(net, skb, &flow_keys_basic_dissector, flow,
                                  data, proto, nhoff, hlen, flags);
}

void skb_flow_dissect_meta(const struct sk_buff *skb,
                           struct flow_dissector *flow_dissector,
                           void *target_container);

/* Gets a skb connection tracking info, ctinfo map should be a
 * map of mapsize to translate enum ip_conntrack_info states
 * to user states.
 */
void
skb_flow_dissect_ct(const struct sk_buff *skb,
                    struct flow_dissector *flow_dissector,
                    void *target_container,
                    u16 *ctinfo_map, size_t mapsize,
                    bool post_ct, u16 zone);
void
skb_flow_dissect_tunnel_info(const struct sk_buff *skb,
                             struct flow_dissector *flow_dissector,
                             void *target_container);

void skb_flow_dissect_hash(const struct sk_buff *skb,
                           struct flow_dissector *flow_dissector,
                           void *target_container);

static inline __u32 skb_get_hash_net(const struct net *net, struct sk_buff *skb)
{
        if (!skb->l4_hash && !skb->sw_hash)
                __skb_get_hash_net(net, skb);

        return skb->hash;
}

static inline __u32 skb_get_hash(struct sk_buff *skb)
{
        if (!skb->l4_hash && !skb->sw_hash)
                __skb_get_hash_net(NULL, skb);

        return skb->hash;
}

static inline __u32 skb_get_hash_flowi6(struct sk_buff *skb, const struct flowi6 *fl6)
{
        if (!skb->l4_hash && !skb->sw_hash) {
                struct flow_keys keys;
                __u32 hash = __get_hash_from_flowi6(fl6, &keys);

                __skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys));
        }

        return skb->hash;
}

__u32 skb_get_hash_perturb(const struct sk_buff *skb,
                           const siphash_key_t *perturb);

static inline __u32 skb_get_hash_raw(const struct sk_buff *skb)
{
        return skb->hash;
}

static inline void skb_copy_hash(struct sk_buff *to, const struct sk_buff *from)
{
        to->hash = from->hash;
        to->sw_hash = from->sw_hash;
        to->l4_hash = from->l4_hash;
};

static inline int skb_cmp_decrypted(const struct sk_buff *skb1,
                                    const struct sk_buff *skb2)
{
#ifdef CONFIG_SKB_DECRYPTED
        return skb2->decrypted - skb1->decrypted;
#else
        return 0;
#endif
}

static inline bool skb_is_decrypted(const struct sk_buff *skb)
{
#ifdef CONFIG_SKB_DECRYPTED
        return skb->decrypted;
#else
        return false;
#endif
}

static inline void skb_copy_decrypted(struct sk_buff *to,
                                      const struct sk_buff *from)
{
#ifdef CONFIG_SKB_DECRYPTED
        to->decrypted = from->decrypted;
#endif
}

#ifdef NET_SKBUFF_DATA_USES_OFFSET
static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
{
        return skb->head + skb->end;
}

static inline unsigned int skb_end_offset(const struct sk_buff *skb)
{
        return skb->end;
}

static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset)
{
        skb->end = offset;
}
#else
static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
{
        return skb->end;
}

static inline unsigned int skb_end_offset(const struct sk_buff *skb)
{
        return skb->end - skb->head;
}

static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset)
{
        skb->end = skb->head + offset;
}
#endif

extern const struct ubuf_info_ops msg_zerocopy_ubuf_ops;

struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
                                       struct ubuf_info *uarg);

void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref);

int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk,
                            struct sk_buff *skb, struct iov_iter *from,
                            size_t length);

int zerocopy_fill_skb_from_iter(struct sk_buff *skb,
                                struct iov_iter *from, size_t length);

static inline int skb_zerocopy_iter_dgram(struct sk_buff *skb,
                                          struct msghdr *msg, int len)
{
        return __zerocopy_sg_from_iter(msg, skb->sk, skb, &msg->msg_iter, len);
}

int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
                             struct msghdr *msg, int len,
                             struct ubuf_info *uarg);

/* Internal */
#define skb_shinfo(SKB)        ((struct skb_shared_info *)(skb_end_pointer(SKB)))

static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb)
{
        return &skb_shinfo(skb)->hwtstamps;
}

static inline struct ubuf_info *skb_zcopy(struct sk_buff *skb)
{
        bool is_zcopy = skb && skb_shinfo(skb)->flags & SKBFL_ZEROCOPY_ENABLE;

        return is_zcopy ? skb_uarg(skb) : NULL;
}

static inline bool skb_zcopy_pure(const struct sk_buff *skb)
{
        return skb_shinfo(skb)->flags & SKBFL_PURE_ZEROCOPY;
}

static inline bool skb_zcopy_managed(const struct sk_buff *skb)
{
        return skb_shinfo(skb)->flags & SKBFL_MANAGED_FRAG_REFS;
}

static inline bool skb_pure_zcopy_same(const struct sk_buff *skb1,
                                       const struct sk_buff *skb2)
{
        return skb_zcopy_pure(skb1) == skb_zcopy_pure(skb2);
}

static inline void net_zcopy_get(struct ubuf_info *uarg)
{
        refcount_inc(&uarg->refcnt);
}

static inline void skb_zcopy_init(struct sk_buff *skb, struct ubuf_info *uarg)
{
        skb_shinfo(skb)->destructor_arg = uarg;
        skb_shinfo(skb)->flags |= uarg->flags;
}

static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg,
                                 bool *have_ref)
{
        if (skb && uarg && !skb_zcopy(skb)) {
                if (unlikely(have_ref && *have_ref))
                        *have_ref = false;
                else
                        net_zcopy_get(uarg);
                skb_zcopy_init(skb, uarg);
        }
}

static inline void skb_zcopy_set_nouarg(struct sk_buff *skb, void *val)
{
        skb_shinfo(skb)->destructor_arg = (void *)((uintptr_t) val | 0x1UL);
        skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_FRAG;
}

static inline bool skb_zcopy_is_nouarg(struct sk_buff *skb)
{
        return (uintptr_t) skb_shinfo(skb)->destructor_arg & 0x1UL;
}

static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb)
{
        return (void *)((uintptr_t) skb_shinfo(skb)->destructor_arg & ~0x1UL);
}

static inline void net_zcopy_put(struct ubuf_info *uarg)
{
        if (uarg)
                uarg->ops->complete(NULL, uarg, true);
}

static inline void net_zcopy_put_abort(struct ubuf_info *uarg, bool have_uref)
{
        if (uarg) {
                if (uarg->ops == &msg_zerocopy_ubuf_ops)
                        msg_zerocopy_put_abort(uarg, have_uref);
                else if (have_uref)
                        net_zcopy_put(uarg);
        }
}

/* Release a reference on a zerocopy structure */
static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success)
{
        struct ubuf_info *uarg = skb_zcopy(skb);

        if (uarg) {
                if (!skb_zcopy_is_nouarg(skb))
                        uarg->ops->complete(skb, uarg, zerocopy_success);

                skb_shinfo(skb)->flags &= ~SKBFL_ALL_ZEROCOPY;
        }
}

void __skb_zcopy_downgrade_managed(struct sk_buff *skb);

static inline void skb_zcopy_downgrade_managed(struct sk_buff *skb)
{
        if (unlikely(skb_zcopy_managed(skb)))
                __skb_zcopy_downgrade_managed(skb);
}

/* Return true if frags in this skb are readable by the host. */
static inline bool skb_frags_readable(const struct sk_buff *skb)
{
        return !skb->unreadable;
}

static inline void skb_mark_not_on_list(struct sk_buff *skb)
{
        skb->next = NULL;
}

static inline void skb_poison_list(struct sk_buff *skb)
{
#ifdef CONFIG_DEBUG_NET
        skb->next = SKB_LIST_POISON_NEXT;
#endif
}

/* Iterate through singly-linked GSO fragments of an skb. */
#define skb_list_walk_safe(first, skb, next_skb)                               \
        for ((skb) = (first), (next_skb) = (skb) ? (skb)->next : NULL; (skb);  \
             (skb) = (next_skb), (next_skb) = (skb) ? (skb)->next : NULL)

static inline void skb_list_del_init(struct sk_buff *skb)
{
        __list_del_entry(&skb->list);
        skb_mark_not_on_list(skb);
}

/**
 *        skb_queue_empty - check if a queue is empty
 *        @list: queue head
 *
 *        Returns true if the queue is empty, false otherwise.
 */
static inline int skb_queue_empty(const struct sk_buff_head *list)
{
        return list->next == (const struct sk_buff *) list;
}

/**
 *        skb_queue_empty_lockless - check if a queue is empty
 *        @list: queue head
 *
 *        Returns true if the queue is empty, false otherwise.
 *        This variant can be used in lockless contexts.
 */
static inline bool skb_queue_empty_lockless(const struct sk_buff_head *list)
{
        return READ_ONCE(list->next) == (const struct sk_buff *) list;
}


/**
 *        skb_queue_is_last - check if skb is the last entry in the queue
 *        @list: queue head
 *        @skb: buffer
 *
 *        Returns true if @skb is the last buffer on the list.
 */
static inline bool skb_queue_is_last(const struct sk_buff_head *list,
                                     const struct sk_buff *skb)
{
        return skb->next == (const struct sk_buff *) list;
}

/**
 *        skb_queue_is_first - check if skb is the first entry in the queue
 *        @list: queue head
 *        @skb: buffer
 *
 *        Returns true if @skb is the first buffer on the list.
 */
static inline bool skb_queue_is_first(const struct sk_buff_head *list,
                                      const struct sk_buff *skb)
{
        return skb->prev == (const struct sk_buff *) list;
}

/**
 *        skb_queue_next - return the next packet in the queue
 *        @list: queue head
 *        @skb: current buffer
 *
 *        Return the next packet in @list after @skb.  It is only valid to
 *        call this if skb_queue_is_last() evaluates to false.
 */
static inline struct sk_buff *skb_queue_next(const struct sk_buff_head *list,
                                             const struct sk_buff *skb)
{
        /* This BUG_ON may seem severe, but if we just return then we
         * are going to dereference garbage.
         */
        BUG_ON(skb_queue_is_last(list, skb));
        return skb->next;
}

/**
 *        skb_queue_prev - return the prev packet in the queue
 *        @list: queue head
 *        @skb: current buffer
 *
 *        Return the prev packet in @list before @skb.  It is only valid to
 *        call this if skb_queue_is_first() evaluates to false.
 */
static inline struct sk_buff *skb_queue_prev(const struct sk_buff_head *list,
                                             const struct sk_buff *skb)
{
        /* This BUG_ON may seem severe, but if we just return then we
         * are going to dereference garbage.
         */
        BUG_ON(skb_queue_is_first(list, skb));
        return skb->prev;
}

/**
 *        skb_get - reference buffer
 *        @skb: buffer to reference
 *
 *        Makes another reference to a socket buffer and returns a pointer
 *        to the buffer.
 */
static inline struct sk_buff *skb_get(struct sk_buff *skb)
{
        refcount_inc(&skb->users);
        return skb;
}

/*
 * If users == 1, we are the only owner and can avoid redundant atomic changes.
 */

/**
 *        skb_cloned - is the buffer a clone
 *        @skb: buffer to check
 *
 *        Returns true if the buffer was generated with skb_clone() and is
 *        one of multiple shared copies of the buffer. Cloned buffers are
 *        shared data so must not be written to under normal circumstances.
 */
static inline int skb_cloned(const struct sk_buff *skb)
{
        return skb->cloned &&
               (atomic_read(&skb_shinfo(skb)->dataref) & SKB_DATAREF_MASK) != 1;
}

static inline int skb_unclone(struct sk_buff *skb, gfp_t pri)
{
        might_sleep_if(gfpflags_allow_blocking(pri));

        if (skb_cloned(skb))
                return pskb_expand_head(skb, 0, 0, pri);

        return 0;
}

/* This variant of skb_unclone() makes sure skb->truesize
 * and skb_end_offset() are not changed, whenever a new skb->head is needed.
 *
 * Indeed there is no guarantee that ksize(kmalloc(X)) == ksize(kmalloc(X))
 * when various debugging features are in place.
 */
int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri);
static inline int skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri)
{
        might_sleep_if(gfpflags_allow_blocking(pri));

        if (skb_cloned(skb))
                return __skb_unclone_keeptruesize(skb, pri);
        return 0;
}

/**
 *        skb_header_cloned - is the header a clone
 *        @skb: buffer to check
 *
 *        Returns true if modifying the header part of the buffer requires
 *        the data to be copied.
 */
static inline int skb_header_cloned(const struct sk_buff *skb)
{
        int dataref;

        if (!skb->cloned)
                return 0;

        dataref = atomic_read(&skb_shinfo(skb)->dataref);
        dataref = (dataref & SKB_DATAREF_MASK) - (dataref >> SKB_DATAREF_SHIFT);
        return dataref != 1;
}

static inline int skb_header_unclone(struct sk_buff *skb, gfp_t pri)
{
        might_sleep_if(gfpflags_allow_blocking(pri));

        if (skb_header_cloned(skb))
                return pskb_expand_head(skb, 0, 0, pri);

        return 0;
}

/**
 * __skb_header_release() - allow clones to use the headroom
 * @skb: buffer to operate on
 *
 * See "DOC: dataref and headerless skbs".
 */
static inline void __skb_header_release(struct sk_buff *skb)
{
        skb->nohdr = 1;
        atomic_set(&skb_shinfo(skb)->dataref, 1 + (1 << SKB_DATAREF_SHIFT));
}


/**
 *        skb_shared - is the buffer shared
 *        @skb: buffer to check
 *
 *        Returns true if more than one person has a reference to this
 *        buffer.
 */
static inline int skb_shared(const struct sk_buff *skb)
{
        return refcount_read(&skb->users) != 1;
}

/**
 *        skb_share_check - check if buffer is shared and if so clone it
 *        @skb: buffer to check
 *        @pri: priority for memory allocation
 *
 *        If the buffer is shared the buffer is cloned and the old copy
 *        drops a reference. A new clone with a single reference is returned.
 *        If the buffer is not shared the original buffer is returned. When
 *        being called from interrupt status or with spinlocks held pri must
 *        be GFP_ATOMIC.
 *
 *        NULL is returned on a memory allocation failure.
 */
static inline struct sk_buff *skb_share_check(struct sk_buff *skb, gfp_t pri)
{
        might_sleep_if(gfpflags_allow_blocking(pri));
        if (skb_shared(skb)) {
                struct sk_buff *nskb = skb_clone(skb, pri);

                if (likely(nskb))
                        consume_skb(skb);
                else
                        kfree_skb(skb);
                skb = nskb;
        }
        return skb;
}

/*
 *        Copy shared buffers into a new sk_buff. We effectively do COW on
 *        packets to handle cases where we have a local reader and forward
 *        and a couple of other messy ones. The normal one is tcpdumping
 *        a packet that's being forwarded.
 */

/**
 *        skb_unshare - make a copy of a shared buffer
 *        @skb: buffer to check
 *        @pri: priority for memory allocation
 *
 *        If the socket buffer is a clone then this function creates a new
 *        copy of the data, drops a reference count on the old copy and returns
 *        the new copy with the reference count at 1. If the buffer is not a clone
 *        the original buffer is returned. When called with a spinlock held or
 *        from interrupt state @pri must be %GFP_ATOMIC
 *
 *        %NULL is returned on a memory allocation failure.
 */
static inline struct sk_buff *skb_unshare(struct sk_buff *skb,
                                          gfp_t pri)
{
        might_sleep_if(gfpflags_allow_blocking(pri));
        if (skb_cloned(skb)) {
                struct sk_buff *nskb = skb_copy(skb, pri);

                /* Free our shared copy */
                if (likely(nskb))
                        consume_skb(skb);
                else
                        kfree_skb(skb);
                skb = nskb;
        }
        return skb;
}

/**
 *        skb_peek - peek at the head of an &sk_buff_head
 *        @list_: list to peek at
 *
 *        Peek an &sk_buff. Unlike most other operations you _MUST_
 *        be careful with this one. A peek leaves the buffer on the
 *        list and someone else may run off with it. You must hold
 *        the appropriate locks or have a private queue to do this.
 *
 *        Returns %NULL for an empty list or a pointer to the head element.
 *        The reference count is not incremented and the reference is therefore
 *        volatile. Use with caution.
 */
static inline struct sk_buff *skb_peek(const struct sk_buff_head *list_)
{
        struct sk_buff *skb = list_->next;

        if (skb == (struct sk_buff *)list_)
                skb = NULL;
        return skb;
}

/**
 *        __skb_peek - peek at the head of a non-empty &sk_buff_head
 *        @list_: list to peek at
 *
 *        Like skb_peek(), but the caller knows that the list is not empty.
 */
static inline struct sk_buff *__skb_peek(const struct sk_buff_head *list_)
{
        return list_->next;
}

/**
 *        skb_peek_next - peek skb following the given one from a queue
 *        @skb: skb to start from
 *        @list_: list to peek at
 *
 *        Returns %NULL when the end of the list is met or a pointer to the
 *        next element. The reference count is not incremented and the
 *        reference is therefore volatile. Use with caution.
 */
static inline struct sk_buff *skb_peek_next(struct sk_buff *skb,
                const struct sk_buff_head *list_)
{
        struct sk_buff *next = skb->next;

        if (next == (struct sk_buff *)list_)
                next = NULL;
        return next;
}

/**
 *        skb_peek_tail - peek at the tail of an &sk_buff_head
 *        @list_: list to peek at
 *
 *        Peek an &sk_buff. Unlike most other operations you _MUST_
 *        be careful with this one. A peek leaves the buffer on the
 *        list and someone else may run off with it. You must hold
 *        the appropriate locks or have a private queue to do this.
 *
 *        Returns %NULL for an empty list or a pointer to the tail element.
 *        The reference count is not incremented and the reference is therefore
 *        volatile. Use with caution.
 */
static inline struct sk_buff *skb_peek_tail(const struct sk_buff_head *list_)
{
        struct sk_buff *skb = READ_ONCE(list_->prev);

        if (skb == (struct sk_buff *)list_)
                skb = NULL;
        return skb;

}

/**
 *        skb_queue_len        - get queue length
 *        @list_: list to measure
 *
 *        Return the length of an &sk_buff queue.
 */
static inline __u32 skb_queue_len(const struct sk_buff_head *list_)
{
        return list_->qlen;
}

/**
 *        skb_queue_len_lockless        - get queue length
 *        @list_: list to measure
 *
 *        Return the length of an &sk_buff queue.
 *        This variant can be used in lockless contexts.
 */
static inline __u32 skb_queue_len_lockless(const struct sk_buff_head *list_)
{
        return READ_ONCE(list_->qlen);
}

/**
 *        __skb_queue_head_init - initialize non-spinlock portions of sk_buff_head
 *        @list: queue to initialize
 *
 *        This initializes only the list and queue length aspects of
 *        an sk_buff_head object.  This allows to initialize the list
 *        aspects of an sk_buff_head without reinitializing things like
 *        the spinlock.  It can also be used for on-stack sk_buff_head
 *        objects where the spinlock is known to not be used.
 */
static inline void __skb_queue_head_init(struct sk_buff_head *list)
{
        list->prev = list->next = (struct sk_buff *)list;
        list->qlen = 0;
}

/*
 * This function creates a split out lock class for each invocation;
 * this is needed for now since a whole lot of users of the skb-queue
 * infrastructure in drivers have different locking usage (in hardirq)
 * than the networking core (in softirq only). In the long run either the
 * network layer or drivers should need annotation to consolidate the
 * main types of usage into 3 classes.
 */
static inline void skb_queue_head_init(struct sk_buff_head *list)
{
        spin_lock_init(&list->lock);
        __skb_queue_head_init(list);
}

static inline void skb_queue_head_init_class(struct sk_buff_head *list,
                struct lock_class_key *class)
{
        skb_queue_head_init(list);
        lockdep_set_class(&list->lock, class);
}

/*
 *        Insert an sk_buff on a list.
 *
 *        The "__skb_xxxx()" functions are the non-atomic ones that
 *        can only be called with interrupts disabled.
 */
static inline void __skb_insert(struct sk_buff *newsk,
                                struct sk_buff *prev, struct sk_buff *next,
                                struct sk_buff_head *list)
{
        /* See skb_queue_empty_lockless() and skb_peek_tail()
         * for the opposite READ_ONCE()
         */
        WRITE_ONCE(newsk->next, next);
        WRITE_ONCE(newsk->prev, prev);
        WRITE_ONCE(((struct sk_buff_list *)next)->prev, newsk);
        WRITE_ONCE(((struct sk_buff_list *)prev)->next, newsk);
        WRITE_ONCE(list->qlen, list->qlen + 1);
}

static inline void __skb_queue_splice(const struct sk_buff_head *list,
                                      struct sk_buff *prev,
                                      struct sk_buff *next)
{
        struct sk_buff *first = list->next;
        struct sk_buff *last = list->prev;

        WRITE_ONCE(first->prev, prev);
        WRITE_ONCE(prev->next, first);

        WRITE_ONCE(last->next, next);
        WRITE_ONCE(next->prev, last);
}

/**
 *        skb_queue_splice - join two skb lists, this is designed for stacks
 *        @list: the new list to add
 *        @head: the place to add it in the first list
 */
static inline void skb_queue_splice(const struct sk_buff_head *list,
                                    struct sk_buff_head *head)
{
        if (!skb_queue_empty(list)) {
                __skb_queue_splice(list, (struct sk_buff *) head, head->next);
                head->qlen += list->qlen;
        }
}

/**
 *        skb_queue_splice_init - join two skb lists and reinitialise the emptied list
 *        @list: the new list to add
 *        @head: the place to add it in the first list
 *
 *        The list at @list is reinitialised
 */
static inline void skb_queue_splice_init(struct sk_buff_head *list,
                                         struct sk_buff_head *head)
{
        if (!skb_queue_empty(list)) {
                __skb_queue_splice(list, (struct sk_buff *) head, head->next);
                head->qlen += list->qlen;
                __skb_queue_head_init(list);
        }
}

/**
 *        skb_queue_splice_tail - join two skb lists, each list being a queue
 *        @list: the new list to add
 *        @head: the place to add it in the first list
 */
static inline void skb_queue_splice_tail(const struct sk_buff_head *list,
                                         struct sk_buff_head *head)
{
        if (!skb_queue_empty(list)) {
                __skb_queue_splice(list, head->prev, (struct sk_buff *) head);
                head->qlen += list->qlen;
        }
}

/**
 *        skb_queue_splice_tail_init - join two skb lists and reinitialise the emptied list
 *        @list: the new list to add
 *        @head: the place to add it in the first list
 *
 *        Each of the lists is a queue.
 *        The list at @list is reinitialised
 */
static inline void skb_queue_splice_tail_init(struct sk_buff_head *list,
                                              struct sk_buff_head *head)
{
        if (!skb_queue_empty(list)) {
                __skb_queue_splice(list, head->prev, (struct sk_buff *) head);
                head->qlen += list->qlen;
                __skb_queue_head_init(list);
        }
}

/**
 *        __skb_queue_after - queue a buffer at the list head
 *        @list: list to use
 *        @prev: place after this buffer
 *        @newsk: buffer to queue
 *
 *        Queue a buffer int the middle of a list. This function takes no locks
 *        and you must therefore hold required locks before calling it.
 *
 *        A buffer cannot be placed on two lists at the same time.
 */
static inline void __skb_queue_after(struct sk_buff_head *list,
                                     struct sk_buff *prev,
                                     struct sk_buff *newsk)
{
        __skb_insert(newsk, prev, ((struct sk_buff_list *)prev)->next, list);
}

void skb_append(struct sk_buff *old, struct sk_buff *newsk,
                struct sk_buff_head *list);

static inline void __skb_queue_before(struct sk_buff_head *list,
                                      struct sk_buff *next,
                                      struct sk_buff *newsk)
{
        __skb_insert(newsk, ((struct sk_buff_list *)next)->prev, next, list);
}

/**
 *        __skb_queue_head - queue a buffer at the list head
 *        @list: list to use
 *        @newsk: buffer to queue
 *
 *        Queue a buffer at the start of a list. This function takes no locks
 *        and you must therefore hold required locks before calling it.
 *
 *        A buffer cannot be placed on two lists at the same time.
 */
static inline void __skb_queue_head(struct sk_buff_head *list,
                                    struct sk_buff *newsk)
{
        __skb_queue_after(list, (struct sk_buff *)list, newsk);
}
void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk);

/**
 *        __skb_queue_tail - queue a buffer at the list tail
 *        @list: list to use
 *        @newsk: buffer to queue
 *
 *        Queue a buffer at the end of a list. This function takes no locks
 *        and you must therefore hold required locks before calling it.
 *
 *        A buffer cannot be placed on two lists at the same time.
 */
static inline void __skb_queue_tail(struct sk_buff_head *list,
                                   struct sk_buff *newsk)
{
        __skb_queue_before(list, (struct sk_buff *)list, newsk);
}
void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk);

/*
 * remove sk_buff from list. _Must_ be called atomically, and with
 * the list known..
 */
void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list);
static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
{
        struct sk_buff *next, *prev;

        WRITE_ONCE(list->qlen, list->qlen - 1);
        next           = skb->next;
        prev           = skb->prev;
        skb->next  = skb->prev = NULL;
        WRITE_ONCE(next->prev, prev);
        WRITE_ONCE(prev->next, next);
}

/**
 *        __skb_dequeue - remove from the head of the queue
 *        @list: list to dequeue from
 *
 *        Remove the head of the list. This function does not take any locks
 *        so must be used with appropriate locks held only. The head item is
 *        returned or %NULL if the list is empty.
 */
static inline struct sk_buff *__skb_dequeue(struct sk_buff_head *list)
{
        struct sk_buff *skb = skb_peek(list);
        if (skb)
                __skb_unlink(skb, list);
        return skb;
}
struct sk_buff *skb_dequeue(struct sk_buff_head *list);

/**
 *        __skb_dequeue_tail - remove from the tail of the queue
 *        @list: list to dequeue from
 *
 *        Remove the tail of the list. This function does not take any locks
 *        so must be used with appropriate locks held only. The tail item is
 *        returned or %NULL if the list is empty.
 */
static inline struct sk_buff *__skb_dequeue_tail(struct sk_buff_head *list)
{
        struct sk_buff *skb = skb_peek_tail(list);
        if (skb)
                __skb_unlink(skb, list);
        return skb;
}
struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list);


static inline bool skb_is_nonlinear(const struct sk_buff *skb)
{
        return skb->data_len;
}

static inline unsigned int skb_headlen(const struct sk_buff *skb)
{
        return skb->len - skb->data_len;
}

static inline unsigned int __skb_pagelen(const struct sk_buff *skb)
{
        unsigned int i, len = 0;

        for (i = skb_shinfo(skb)->nr_frags - 1; (int)i >= 0; i--)
                len += skb_frag_size(&skb_shinfo(skb)->frags[i]);
        return len;
}

static inline unsigned int skb_pagelen(const struct sk_buff *skb)
{
        return skb_headlen(skb) + __skb_pagelen(skb);
}

static inline void skb_frag_fill_netmem_desc(skb_frag_t *frag,
                                             netmem_ref netmem, int off,
                                             int size)
{
        frag->netmem = netmem;
        frag->offset = off;
        skb_frag_size_set(frag, size);
}

static inline void skb_frag_fill_page_desc(skb_frag_t *frag,
                                           struct page *page,
                                           int off, int size)
{
        skb_frag_fill_netmem_desc(frag, page_to_netmem(page), off, size);
}

static inline void __skb_fill_netmem_desc_noacc(struct skb_shared_info *shinfo,
                                                int i, netmem_ref netmem,
                                                int off, int size)
{
        skb_frag_t *frag = &shinfo->frags[i];

        skb_frag_fill_netmem_desc(frag, netmem, off, size);
}

static inline void __skb_fill_page_desc_noacc(struct skb_shared_info *shinfo,
                                              int i, struct page *page,
                                              int off, int size)
{
        __skb_fill_netmem_desc_noacc(shinfo, i, page_to_netmem(page), off,
                                     size);
}

/**
 * skb_len_add - adds a number to len fields of skb
 * @skb: buffer to add len to
 * @delta: number of bytes to add
 */
static inline void skb_len_add(struct sk_buff *skb, int delta)
{
        skb->len += delta;
        skb->data_len += delta;
        skb->truesize += delta;
}

/**
 * __skb_fill_netmem_desc - initialise a fragment in an skb
 * @skb: buffer containing fragment to be initialised
 * @i: fragment index to initialise
 * @netmem: the netmem to use for this fragment
 * @off: the offset to the data with @page
 * @size: the length of the data
 *
 * Initialises the @i'th fragment of @skb to point to &size bytes at
 * offset @off within @page.
 *
 * Does not take any additional reference on the fragment.
 */
static inline void __skb_fill_netmem_desc(struct sk_buff *skb, int i,
                                          netmem_ref netmem, int off, int size)
{
        struct page *page;

        __skb_fill_netmem_desc_noacc(skb_shinfo(skb), i, netmem, off, size);

        if (netmem_is_net_iov(netmem)) {
                skb->unreadable = true;
                return;
        }

        page = netmem_to_page(netmem);

        /* Propagate page pfmemalloc to the skb if we can. The problem is
         * that not all callers have unique ownership of the page but rely
         * on page_is_pfmemalloc doing the right thing(tm).
         */
        page = compound_head(page);
        if (page_is_pfmemalloc(page))
                skb->pfmemalloc = true;
}

static inline void __skb_fill_page_desc(struct sk_buff *skb, int i,
                                        struct page *page, int off, int size)
{
        __skb_fill_netmem_desc(skb, i, page_to_netmem(page), off, size);
}

static inline void skb_fill_netmem_desc(struct sk_buff *skb, int i,
                                        netmem_ref netmem, int off, int size)
{
        __skb_fill_netmem_desc(skb, i, netmem, off, size);
        skb_shinfo(skb)->nr_frags = i + 1;
}

/**
 * skb_fill_page_desc - initialise a paged fragment in an skb
 * @skb: buffer containing fragment to be initialised
 * @i: paged fragment index to initialise
 * @page: the page to use for this fragment
 * @off: the offset to the data with @page
 * @size: the length of the data
 *
 * As per __skb_fill_page_desc() -- initialises the @i'th fragment of
 * @skb to point to @size bytes at offset @off within @page. In
 * addition updates @skb such that @i is the last fragment.
 *
 * Does not take any additional reference on the fragment.
 */
static inline void skb_fill_page_desc(struct sk_buff *skb, int i,
                                      struct page *page, int off, int size)
{
        skb_fill_netmem_desc(skb, i, page_to_netmem(page), off, size);
}

/**
 * skb_fill_page_desc_noacc - initialise a paged fragment in an skb
 * @skb: buffer containing fragment to be initialised
 * @i: paged fragment index to initialise
 * @page: the page to use for this fragment
 * @off: the offset to the data with @page
 * @size: the length of the data
 *
 * Variant of skb_fill_page_desc() which does not deal with
 * pfmemalloc, if page is not owned by us.
 */
static inline void skb_fill_page_desc_noacc(struct sk_buff *skb, int i,
                                            struct page *page, int off,
                                            int size)
{
        struct skb_shared_info *shinfo = skb_shinfo(skb);

        __skb_fill_page_desc_noacc(shinfo, i, page, off, size);
        shinfo->nr_frags = i + 1;
}

void skb_add_rx_frag_netmem(struct sk_buff *skb, int i, netmem_ref netmem,
                            int off, int size, unsigned int truesize);

static inline void skb_add_rx_frag(struct sk_buff *skb, int i,
                                   struct page *page, int off, int size,
                                   unsigned int truesize)
{
        skb_add_rx_frag_netmem(skb, i, page_to_netmem(page), off, size,
                               truesize);
}

void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
                          unsigned int truesize);

#define SKB_LINEAR_ASSERT(skb)  BUG_ON(skb_is_nonlinear(skb))

#ifdef NET_SKBUFF_DATA_USES_OFFSET
static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
{
        return skb->head + skb->tail;
}

static inline void skb_reset_tail_pointer(struct sk_buff *skb)
{
        skb->tail = skb->data - skb->head;
}

static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
{
        skb_reset_tail_pointer(skb);
        skb->tail += offset;
}

#else /* NET_SKBUFF_DATA_USES_OFFSET */
static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
{
        return skb->tail;
}

static inline void skb_reset_tail_pointer(struct sk_buff *skb)
{
        skb->tail = skb->data;
}

static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
{
        skb->tail = skb->data + offset;
}

#endif /* NET_SKBUFF_DATA_USES_OFFSET */

static inline void skb_assert_len(struct sk_buff *skb)
{
#ifdef CONFIG_DEBUG_NET
        if (WARN_ONCE(!skb->len, "%s\n", __func__))
                DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
#endif /* CONFIG_DEBUG_NET */
}

#if defined(CONFIG_FAIL_SKB_REALLOC)
void skb_might_realloc(struct sk_buff *skb);
#else
static inline void skb_might_realloc(struct sk_buff *skb) {}
#endif

/*
 *        Add data to an sk_buff
 */
void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len);
void *skb_put(struct sk_buff *skb, unsigned int len);
static inline void *__skb_put(struct sk_buff *skb, unsigned int len)
{
        void *tmp = skb_tail_pointer(skb);
        SKB_LINEAR_ASSERT(skb);
        skb->tail += len;
        skb->len  += len;
        return tmp;
}

static inline void *__skb_put_zero(struct sk_buff *skb, unsigned int len)
{
        void *tmp = __skb_put(skb, len);

        memset(tmp, 0, len);
        return tmp;
}

static inline void *__skb_put_data(struct sk_buff *skb, const void *data,
                                   unsigned int len)
{
        void *tmp = __skb_put(skb, len);

        memcpy(tmp, data, len);
        return tmp;
}

static inline void __skb_put_u8(struct sk_buff *skb, u8 val)
{
        *(u8 *)__skb_put(skb, 1) = val;
}

static inline void *skb_put_zero(struct sk_buff *skb, unsigned int len)
{
        void *tmp = skb_put(skb, len);

        memset(tmp, 0, len);

        return tmp;
}

static inline void *skb_put_data(struct sk_buff *skb, const void *data,
                                 unsigned int len)
{
        void *tmp = skb_put(skb, len);

        memcpy(tmp, data, len);

        return tmp;
}

static inline void skb_put_u8(struct sk_buff *skb, u8 val)
{
        *(u8 *)skb_put(skb, 1) = val;
}

void *skb_push(struct sk_buff *skb, unsigned int len);
static inline void *__skb_push(struct sk_buff *skb, unsigned int len)
{
        DEBUG_NET_WARN_ON_ONCE(len > INT_MAX);

        skb->data -= len;
        skb->len  += len;
        return skb->data;
}

void *skb_pull(struct sk_buff *skb, unsigned int len);
static inline void *__skb_pull(struct sk_buff *skb, unsigned int len)
{
        DEBUG_NET_WARN_ON_ONCE(len > INT_MAX);

        skb->len -= len;
        if (unlikely(skb->len < skb->data_len)) {
#if defined(CONFIG_DEBUG_NET)
                skb->len += len;
                pr_err("__skb_pull(len=%u)\n", len);
                skb_dump(KERN_ERR, skb, false);
#endif
                BUG();
        }
        return skb->data += len;
}

static inline void *skb_pull_inline(struct sk_buff *skb, unsigned int len)
{
        return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
}

void *skb_pull_data(struct sk_buff *skb, size_t len);

void *__pskb_pull_tail(struct sk_buff *skb, int delta);

static inline enum skb_drop_reason
pskb_may_pull_reason(struct sk_buff *skb, unsigned int len)
{
        DEBUG_NET_WARN_ON_ONCE(len > INT_MAX);
        skb_might_realloc(skb);

        if (likely(len <= skb_headlen(skb)))
                return SKB_NOT_DROPPED_YET;

        if (unlikely(len > skb->len))
                return SKB_DROP_REASON_PKT_TOO_SMALL;

        if (unlikely(!__pskb_pull_tail(skb, len - skb_headlen(skb))))
                return SKB_DROP_REASON_NOMEM;

        return SKB_NOT_DROPPED_YET;
}

static inline bool pskb_may_pull(struct sk_buff *skb, unsigned int len)
{
        return pskb_may_pull_reason(skb, len) == SKB_NOT_DROPPED_YET;
}

static inline void *pskb_pull(struct sk_buff *skb, unsigned int len)
{
        if (!pskb_may_pull(skb, len))
                return NULL;

        skb->len -= len;
        return skb->data += len;
}

void skb_condense(struct sk_buff *skb);

/**
 *        skb_headroom - bytes at buffer head
 *        @skb: buffer to check
 *
 *        Return the number of bytes of free space at the head of an &sk_buff.
 */
static inline unsigned int skb_headroom(const struct sk_buff *skb)
{
        return skb->data - skb->head;
}

/**
 *        skb_tailroom - bytes at buffer end
 *        @skb: buffer to check
 *
 *        Return the number of bytes of free space at the tail of an sk_buff
 */
static inline int skb_tailroom(const struct sk_buff *skb)
{
        return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail;
}

/**
 *        skb_availroom - bytes at buffer end
 *        @skb: buffer to check
 *
 *        Return the number of bytes of free space at the tail of an sk_buff
 *        allocated by sk_stream_alloc()
 */
static inline int skb_availroom(const struct sk_buff *skb)
{
        if (skb_is_nonlinear(skb))
                return 0;

        return skb->end - skb->tail - skb->reserved_tailroom;
}

/**
 *        skb_reserve - adjust headroom
 *        @skb: buffer to alter
 *        @len: bytes to move
 *
 *        Increase the headroom of an empty &sk_buff by reducing the tail
 *        room. This is only allowed for an empty buffer.
 */
static inline void skb_reserve(struct sk_buff *skb, int len)
{
        skb->data += len;
        skb->tail += len;
}

/**
 *        skb_tailroom_reserve - adjust reserved_tailroom
 *        @skb: buffer to alter
 *        @mtu: maximum amount of headlen permitted
 *        @needed_tailroom: minimum amount of reserved_tailroom
 *
 *        Set reserved_tailroom so that headlen can be as large as possible but
 *        not larger than mtu and tailroom cannot be smaller than
 *        needed_tailroom.
 *        The required headroom should already have been reserved before using
 *        this function.
 */
static inline void skb_tailroom_reserve(struct sk_buff *skb, unsigned int mtu,
                                        unsigned int needed_tailroom)
{
        SKB_LINEAR_ASSERT(skb);
        if (mtu < skb_tailroom(skb) - needed_tailroom)
                /* use at most mtu */
                skb->reserved_tailroom = skb_tailroom(skb) - mtu;
        else
                /* use up to all available space */
                skb->reserved_tailroom = needed_tailroom;
}

#define ENCAP_TYPE_ETHER        0
#define ENCAP_TYPE_IPPROTO        1

static inline void skb_set_inner_protocol(struct sk_buff *skb,
                                          __be16 protocol)
{
        skb->inner_protocol = protocol;
        skb->inner_protocol_type = ENCAP_TYPE_ETHER;
}

static inline void skb_set_inner_ipproto(struct sk_buff *skb,
                                         __u8 ipproto)
{
        skb->inner_ipproto = ipproto;
        skb->inner_protocol_type = ENCAP_TYPE_IPPROTO;
}

static inline void skb_reset_inner_headers(struct sk_buff *skb)
{
        skb->inner_mac_header = skb->mac_header;
        skb->inner_network_header = skb->network_header;
        skb->inner_transport_header = skb->transport_header;
}

static inline int skb_mac_header_was_set(const struct sk_buff *skb)
{
        return skb->mac_header != (typeof(skb->mac_header))~0U;
}

static inline void skb_reset_mac_len(struct sk_buff *skb)
{
        if (!skb_mac_header_was_set(skb)) {
                DEBUG_NET_WARN_ON_ONCE(1);
                skb->mac_len = 0;
        } else {
                skb->mac_len = skb->network_header - skb->mac_header;
        }
}

static inline unsigned char *skb_inner_transport_header(const struct sk_buff
                                                        *skb)
{
        return skb->head + skb->inner_transport_header;
}

static inline int skb_inner_transport_offset(const struct sk_buff *skb)
{
        return skb_inner_transport_header(skb) - skb->data;
}

static inline void skb_reset_inner_transport_header(struct sk_buff *skb)
{
        long offset = skb->data - skb->head;

        DEBUG_NET_WARN_ON_ONCE(offset != (typeof(skb->inner_transport_header))offset);
        skb->inner_transport_header = offset;
}

static inline void skb_set_inner_transport_header(struct sk_buff *skb,
                                                   const int offset)
{
        skb_reset_inner_transport_header(skb);
        skb->inner_transport_header += offset;
}

static inline unsigned char *skb_inner_network_header(const struct sk_buff *skb)
{
        return skb->head + skb->inner_network_header;
}

static inline void skb_reset_inner_network_header(struct sk_buff *skb)
{
        long offset = skb->data - skb->head;

        DEBUG_NET_WARN_ON_ONCE(offset != (typeof(skb->inner_network_header))offset);
        skb->inner_network_header = offset;
}

static inline void skb_set_inner_network_header(struct sk_buff *skb,
                                                const int offset)
{
        skb_reset_inner_network_header(skb);
        skb->inner_network_header += offset;
}

static inline bool skb_inner_network_header_was_set(const struct sk_buff *skb)
{
        return skb->inner_network_header > 0;
}

static inline unsigned char *skb_inner_mac_header(const struct sk_buff *skb)
{
        return skb->head + skb->inner_mac_header;
}

static inline void skb_reset_inner_mac_header(struct sk_buff *skb)
{
        long offset = skb->data - skb->head;

        DEBUG_NET_WARN_ON_ONCE(offset != (typeof(skb->inner_mac_header))offset);
        skb->inner_mac_header = offset;
}

static inline void skb_set_inner_mac_header(struct sk_buff *skb,
                                            const int offset)
{
        skb_reset_inner_mac_header(skb);
        skb->inner_mac_header += offset;
}
static inline bool skb_transport_header_was_set(const struct sk_buff *skb)
{
        return skb->transport_header != (typeof(skb->transport_header))~0U;
}

static inline unsigned char *skb_transport_header(const struct sk_buff *skb)
{
        DEBUG_NET_WARN_ON_ONCE(!skb_transport_header_was_set(skb));
        return skb->head + skb->transport_header;
}

static inline void skb_reset_transport_header(struct sk_buff *skb)
{
        long offset = skb->data - skb->head;

        DEBUG_NET_WARN_ON_ONCE(offset != (typeof(skb->transport_header))offset);
        skb->transport_header = offset;
}

static inline void skb_set_transport_header(struct sk_buff *skb,
                                            const int offset)
{
        skb_reset_transport_header(skb);
        skb->transport_header += offset;
}

static inline unsigned char *skb_network_header(const struct sk_buff *skb)
{
        return skb->head + skb->network_header;
}

static inline void skb_reset_network_header(struct sk_buff *skb)
{
        long offset = skb->data - skb->head;

        DEBUG_NET_WARN_ON_ONCE(offset != (typeof(skb->network_header))offset);
        skb->network_header = offset;
}

static inline void skb_set_network_header(struct sk_buff *skb, const int offset)
{
        skb_reset_network_header(skb);
        skb->network_header += offset;
}

static inline unsigned char *skb_mac_header(const struct sk_buff *skb)
{
        DEBUG_NET_WARN_ON_ONCE(!skb_mac_header_was_set(skb));
        return skb->head + skb->mac_header;
}

static inline int skb_mac_offset(const struct sk_buff *skb)
{
        return skb_mac_header(skb) - skb->data;
}

static inline u32 skb_mac_header_len(const struct sk_buff *skb)
{
        DEBUG_NET_WARN_ON_ONCE(!skb_mac_header_was_set(skb));
        return skb->network_header - skb->mac_header;
}

static inline void skb_unset_mac_header(struct sk_buff *skb)
{
        skb->mac_header = (typeof(skb->mac_header))~0U;
}

static inline void skb_reset_mac_header(struct sk_buff *skb)
{
        long offset = skb->data - skb->head;

        DEBUG_NET_WARN_ON_ONCE(offset != (typeof(skb->mac_header))offset);
        skb->mac_header = offset;
}

static inline void skb_set_mac_header(struct sk_buff *skb, const int offset)
{
        skb_reset_mac_header(skb);
        skb->mac_header += offset;
}

static inline void skb_pop_mac_header(struct sk_buff *skb)
{
        skb->mac_header = skb->network_header;
}

static inline void skb_probe_transport_header(struct sk_buff *skb)
{
        struct flow_keys_basic keys;

        if (skb_transport_header_was_set(skb))
                return;

        if (skb_flow_dissect_flow_keys_basic(NULL, skb, &keys,
                                             NULL, 0, 0, 0, 0))
                skb_set_transport_header(skb, keys.control.thoff);
}

static inline void skb_mac_header_rebuild(struct sk_buff *skb)
{
        if (skb_mac_header_was_set(skb)) {
                const unsigned char *old_mac = skb_mac_header(skb);

                skb_set_mac_header(skb, -skb->mac_len);
                memmove(skb_mac_header(skb), old_mac, skb->mac_len);
        }
}

/* Move the full mac header up to current network_header.
 * Leaves skb->data pointing at offset skb->mac_len into the mac_header.
 * Must be provided the complete mac header length.
 */
static inline void skb_mac_header_rebuild_full(struct sk_buff *skb, u32 full_mac_len)
{
        if (skb_mac_header_was_set(skb)) {
                const unsigned char *old_mac = skb_mac_header(skb);

                skb_set_mac_header(skb, -full_mac_len);
                memmove(skb_mac_header(skb), old_mac, full_mac_len);
                __skb_push(skb, full_mac_len - skb->mac_len);
        }
}

static inline int skb_checksum_start_offset(const struct sk_buff *skb)
{
        return skb->csum_start - skb_headroom(skb);
}

static inline unsigned char *skb_checksum_start(const struct sk_buff *skb)
{
        return skb->head + skb->csum_start;
}

static inline int skb_transport_offset(const struct sk_buff *skb)
{
        return skb_transport_header(skb) - skb->data;
}

static inline u32 skb_network_header_len(const struct sk_buff *skb)
{
        DEBUG_NET_WARN_ON_ONCE(!skb_transport_header_was_set(skb));
        return skb->transport_header - skb->network_header;
}

static inline u32 skb_inner_network_header_len(const struct sk_buff *skb)
{
        return skb->inner_transport_header - skb->inner_network_header;
}

static inline int skb_network_offset(const struct sk_buff *skb)
{
        return skb_network_header(skb) - skb->data;
}

static inline int skb_inner_network_offset(const struct sk_buff *skb)
{
        return skb_inner_network_header(skb) - skb->data;
}

static inline enum skb_drop_reason
pskb_network_may_pull_reason(struct sk_buff *skb, unsigned int len)
{
        return pskb_may_pull_reason(skb, skb_network_offset(skb) + len);
}

static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len)
{
        return pskb_network_may_pull_reason(skb, len) == SKB_NOT_DROPPED_YET;
}

/*
 * CPUs often take a performance hit when accessing unaligned memory
 * locations. The actual performance hit varies, it can be small if the
 * hardware handles it or large if we have to take an exception and fix it
 * in software.
 *
 * Since an ethernet header is 14 bytes network drivers often end up with
 * the IP header at an unaligned offset. The IP header can be aligned by
 * shifting the start of the packet by 2 bytes. Drivers should do this
 * with:
 *
 * skb_reserve(skb, NET_IP_ALIGN);
 *
 * The downside to this alignment of the IP header is that the DMA is now
 * unaligned. On some architectures the cost of an unaligned DMA is high
 * and this cost outweighs the gains made by aligning the IP header.
 *
 * Since this trade off varies between architectures, we allow NET_IP_ALIGN
 * to be overridden.
 */
#ifndef NET_IP_ALIGN
#define NET_IP_ALIGN        2
#endif

/*
 * The networking layer reserves some headroom in skb data (via
 * dev_alloc_skb). This is used to avoid having to reallocate skb data when
 * the header has to grow. In the default case, if the header has to grow
 * 32 bytes or less we avoid the reallocation.
 *
 * Unfortunately this headroom changes the DMA alignment of the resulting
 * network packet. As for NET_IP_ALIGN, this unaligned DMA is expensive
 * on some architectures. An architecture can override this value,
 * perhaps setting it to a cacheline in size (since that will maintain
 * cacheline alignment of the DMA). It must be a power of 2.
 *
 * Various parts of the networking layer expect at least 32 bytes of
 * headroom, you should not reduce this.
 *
 * Using max(32, L1_CACHE_BYTES) makes sense (especially with RPS)
 * to reduce average number of cache lines per packet.
 * get_rps_cpu() for example only access one 64 bytes aligned block :
 * NET_IP_ALIGN(2) + ethernet_header(14) + IP_header(20/40) + ports(8)
 */
#ifndef NET_SKB_PAD
#define NET_SKB_PAD        max(32, L1_CACHE_BYTES)
#endif

int ___pskb_trim(struct sk_buff *skb, unsigned int len);

static inline void __skb_set_length(struct sk_buff *skb, unsigned int len)
{
        if (WARN_ON(skb_is_nonlinear(skb)))
                return;
        skb->len = len;
        skb_set_tail_pointer(skb, len);
}

static inline void __skb_trim(struct sk_buff *skb, unsigned int len)
{
        __skb_set_length(skb, len);
}

void skb_trim(struct sk_buff *skb, unsigned int len);

static inline int __pskb_trim(struct sk_buff *skb, unsigned int len)
{
        if (skb->data_len)
                return ___pskb_trim(skb, len);
        __skb_trim(skb, len);
        return 0;
}

static inline int pskb_trim(struct sk_buff *skb, unsigned int len)
{
        skb_might_realloc(skb);
        return (len < skb->len) ? __pskb_trim(skb, len) : 0;
}

/**
 *        pskb_trim_unique - remove end from a paged unique (not cloned) buffer
 *        @skb: buffer to alter
 *        @len: new length
 *
 *        This is identical to pskb_trim except that the caller knows that
 *        the skb is not cloned so we should never get an error due to out-
 *        of-memory.
 */
static inline void pskb_trim_unique(struct sk_buff *skb, unsigned int len)
{
        int err = pskb_trim(skb, len);
        BUG_ON(err);
}

static inline int __skb_grow(struct sk_buff *skb, unsigned int len)
{
        unsigned int diff = len - skb->len;

        if (skb_tailroom(skb) < diff) {
                int ret = pskb_expand_head(skb, 0, diff - skb_tailroom(skb),
                                           GFP_ATOMIC);
                if (ret)
                        return ret;
        }
        __skb_set_length(skb, len);
        return 0;
}

/**
 *        skb_orphan - orphan a buffer
 *        @skb: buffer to orphan
 *
 *        If a buffer currently has an owner then we call the owner's
 *        destructor function and make the @skb unowned. The buffer continues
 *        to exist but is no longer charged to its former owner.
 */
static inline void skb_orphan(struct sk_buff *skb)
{
        if (skb->destructor) {
                skb->destructor(skb);
                skb->destructor = NULL;
                skb->sk                = NULL;
        } else {
                BUG_ON(skb->sk);
        }
}

/**
 *        skb_orphan_frags - orphan the frags contained in a buffer
 *        @skb: buffer to orphan frags from
 *        @gfp_mask: allocation mask for replacement pages
 *
 *        For each frag in the SKB which needs a destructor (i.e. has an
 *        owner) create a copy of that frag and release the original
 *        page by calling the destructor.
 */
static inline int skb_orphan_frags(struct sk_buff *skb, gfp_t gfp_mask)
{
        if (likely(!skb_zcopy(skb)))
                return 0;
        if (skb_shinfo(skb)->flags & SKBFL_DONT_ORPHAN)
                return 0;
        return skb_copy_ubufs(skb, gfp_mask);
}

/* Frags must be orphaned, even if refcounted, if skb might loop to rx path */
static inline int skb_orphan_frags_rx(struct sk_buff *skb, gfp_t gfp_mask)
{
        if (likely(!skb_zcopy(skb)))
                return 0;
        return skb_copy_ubufs(skb, gfp_mask);
}

/**
 *        __skb_queue_purge_reason - empty a list
 *        @list: list to empty
 *        @reason: drop reason
 *
 *        Delete all buffers on an &sk_buff list. Each buffer is removed from
 *        the list and one reference dropped. This function does not take the
 *        list lock and the caller must hold the relevant locks to use it.
 */
static inline void __skb_queue_purge_reason(struct sk_buff_head *list,
                                            enum skb_drop_reason reason)
{
        struct sk_buff *skb;

        while ((skb = __skb_dequeue(list)) != NULL)
                kfree_skb_reason(skb, reason);
}

static inline void __skb_queue_purge(struct sk_buff_head *list)
{
        __skb_queue_purge_reason(list, SKB_DROP_REASON_QUEUE_PURGE);
}

void skb_queue_purge_reason(struct sk_buff_head *list,
                            enum skb_drop_reason reason);

static inline void skb_queue_purge(struct sk_buff_head *list)
{
        skb_queue_purge_reason(list, SKB_DROP_REASON_QUEUE_PURGE);
}

unsigned int skb_rbtree_purge(struct rb_root *root);
void skb_errqueue_purge(struct sk_buff_head *list);

void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask);

/**
 * netdev_alloc_frag - allocate a page fragment
 * @fragsz: fragment size
 *
 * Allocates a frag from a page for receive buffer.
 * Uses GFP_ATOMIC allocations.
 */
static inline void *netdev_alloc_frag(unsigned int fragsz)
{
        return __netdev_alloc_frag_align(fragsz, ~0u);
}

static inline void *netdev_alloc_frag_align(unsigned int fragsz,
                                            unsigned int align)
{
        WARN_ON_ONCE(!is_power_of_2(align));
        return __netdev_alloc_frag_align(fragsz, -align);
}

struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int length,
                                   gfp_t gfp_mask);

/**
 *        netdev_alloc_skb - allocate an skbuff for rx on a specific device
 *        @dev: network device to receive on
 *        @length: length to allocate
 *
 *        Allocate a new &sk_buff and assign it a usage count of one. The
 *        buffer has unspecified headroom built in. Users should allocate
 *        the headroom they think they need without accounting for the
 *        built in space. The built in space is used for optimisations.
 *
 *        %NULL is returned if there is no free memory. Although this function
 *        allocates memory it can be called from an interrupt.
 */
static inline struct sk_buff *netdev_alloc_skb(struct net_device *dev,
                                               unsigned int length)
{
        return __netdev_alloc_skb(dev, length, GFP_ATOMIC);
}

/* legacy helper around __netdev_alloc_skb() */
static inline struct sk_buff *__dev_alloc_skb(unsigned int length,
                                              gfp_t gfp_mask)
{
        return __netdev_alloc_skb(NULL, length, gfp_mask);
}

/* legacy helper around netdev_alloc_skb() */
static inline struct sk_buff *dev_alloc_skb(unsigned int length)
{
        return netdev_alloc_skb(NULL, length);
}


static inline struct sk_buff *__netdev_alloc_skb_ip_align(struct net_device *dev,
                unsigned int length, gfp_t gfp)
{
        struct sk_buff *skb = __netdev_alloc_skb(dev, length + NET_IP_ALIGN, gfp);

        if (NET_IP_ALIGN && skb)
                skb_reserve(skb, NET_IP_ALIGN);
        return skb;
}

static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev,
                unsigned int length)
{
        return __netdev_alloc_skb_ip_align(dev, length, GFP_ATOMIC);
}

static inline void skb_free_frag(void *addr)
{
        page_frag_free(addr);
}

void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask);

static inline void *napi_alloc_frag(unsigned int fragsz)
{
        return __napi_alloc_frag_align(fragsz, ~0u);
}

static inline void *napi_alloc_frag_align(unsigned int fragsz,
                                          unsigned int align)
{
        WARN_ON_ONCE(!is_power_of_2(align));
        return __napi_alloc_frag_align(fragsz, -align);
}

struct sk_buff *napi_alloc_skb(struct napi_struct *napi, unsigned int length);
void napi_consume_skb(struct sk_buff *skb, int budget);

void napi_skb_free_stolen_head(struct sk_buff *skb);
void __napi_kfree_skb(struct sk_buff *skb, enum skb_drop_reason reason);

/**
 * __dev_alloc_pages - allocate page for network Rx
 * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx
 * @order: size of the allocation
 *
 * Allocate a new page.
 *
 * %NULL is returned if there is no free memory.
*/
static inline struct page *__dev_alloc_pages_noprof(gfp_t gfp_mask,
                                             unsigned int order)
{
        /* This piece of code contains several assumptions.
         * 1.  This is for device Rx, therefore a cold page is preferred.
         * 2.  The expectation is the user wants a compound page.
         * 3.  If requesting a order 0 page it will not be compound
         *     due to the check to see if order has a value in prep_new_page
         * 4.  __GFP_MEMALLOC is ignored if __GFP_NOMEMALLOC is set due to
         *     code in gfp_to_alloc_flags that should be enforcing this.
         */
        gfp_mask |= __GFP_COMP | __GFP_MEMALLOC;

        return alloc_pages_node_noprof(NUMA_NO_NODE, gfp_mask, order);
}
#define __dev_alloc_pages(...)        alloc_hooks(__dev_alloc_pages_noprof(__VA_ARGS__))

/*
 * This specialized allocator has to be a macro for its allocations to be
 * accounted separately (to have a separate alloc_tag).
 */
#define dev_alloc_pages(_order) __dev_alloc_pages(GFP_ATOMIC | __GFP_NOWARN, _order)

/**
 * __dev_alloc_page - allocate a page for network Rx
 * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx
 *
 * Allocate a new page.
 *
 * %NULL is returned if there is no free memory.
 */
static inline struct page *__dev_alloc_page_noprof(gfp_t gfp_mask)
{
        return __dev_alloc_pages_noprof(gfp_mask, 0);
}
#define __dev_alloc_page(...)        alloc_hooks(__dev_alloc_page_noprof(__VA_ARGS__))

/*
 * This specialized allocator has to be a macro for its allocations to be
 * accounted separately (to have a separate alloc_tag).
 */
#define dev_alloc_page()        dev_alloc_pages(0)

/**
 * dev_page_is_reusable - check whether a page can be reused for network Rx
 * @page: the page to test
 *
 * A page shouldn't be considered for reusing/recycling if it was allocated
 * under memory pressure or at a distant memory node.
 *
 * Returns: false if this page should be returned to page allocator, true
 * otherwise.
 */
static inline bool dev_page_is_reusable(const struct page *page)
{
        return likely(page_to_nid(page) == numa_mem_id() &&
                      !page_is_pfmemalloc(page));
}

/**
 *        skb_propagate_pfmemalloc - Propagate pfmemalloc if skb is allocated after RX page
 *        @page: The page that was allocated from skb_alloc_page
 *        @skb: The skb that may need pfmemalloc set
 */
static inline void skb_propagate_pfmemalloc(const struct page *page,
                                            struct sk_buff *skb)
{
        if (page_is_pfmemalloc(page))
                skb->pfmemalloc = true;
}

/**
 * skb_frag_off() - Returns the offset of a skb fragment
 * @frag: the paged fragment
 */
static inline unsigned int skb_frag_off(const skb_frag_t *frag)
{
        return frag->offset;
}

/**
 * skb_frag_off_add() - Increments the offset of a skb fragment by @delta
 * @frag: skb fragment
 * @delta: value to add
 */
static inline void skb_frag_off_add(skb_frag_t *frag, int delta)
{
        frag->offset += delta;
}

/**
 * skb_frag_off_set() - Sets the offset of a skb fragment
 * @frag: skb fragment
 * @offset: offset of fragment
 */
static inline void skb_frag_off_set(skb_frag_t *frag, unsigned int offset)
{
        frag->offset = offset;
}

/**
 * skb_frag_off_copy() - Sets the offset of a skb fragment from another fragment
 * @fragto: skb fragment where offset is set
 * @fragfrom: skb fragment offset is copied from
 */
static inline void skb_frag_off_copy(skb_frag_t *fragto,
                                     const skb_frag_t *fragfrom)
{
        fragto->offset = fragfrom->offset;
}

/* Return: true if the skb_frag contains a net_iov. */
static inline bool skb_frag_is_net_iov(const skb_frag_t *frag)
{
        return netmem_is_net_iov(frag->netmem);
}

/**
 * skb_frag_net_iov - retrieve the net_iov referred to by fragment
 * @frag: the fragment
 *
 * Return: the &struct net_iov associated with @frag. Returns NULL if this
 * frag has no associated net_iov.
 */
static inline struct net_iov *skb_frag_net_iov(const skb_frag_t *frag)
{
        if (!skb_frag_is_net_iov(frag))
                return NULL;

        return netmem_to_net_iov(frag->netmem);
}

/**
 * skb_frag_page - retrieve the page referred to by a paged fragment
 * @frag: the paged fragment
 *
 * Return: the &struct page associated with @frag. Returns NULL if this frag
 * has no associated page.
 */
static inline struct page *skb_frag_page(const skb_frag_t *frag)
{
        if (skb_frag_is_net_iov(frag))
                return NULL;

        return netmem_to_page(frag->netmem);
}

/**
 * skb_frag_netmem - retrieve the netmem referred to by a fragment
 * @frag: the fragment
 *
 * Return: the &netmem_ref associated with @frag.
 */
static inline netmem_ref skb_frag_netmem(const skb_frag_t *frag)
{
        return frag->netmem;
}

int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb,
                    unsigned int headroom);
int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb,
                         const struct bpf_prog *prog);

/**
 * skb_frag_address - gets the address of the data contained in a paged fragment
 * @frag: the paged fragment buffer
 *
 * Returns: the address of the data within @frag. The page must already
 * be mapped.
 */
static inline void *skb_frag_address(const skb_frag_t *frag)
{
        if (!skb_frag_page(frag))
                return NULL;

        return page_address(skb_frag_page(frag)) + skb_frag_off(frag);
}

/**
 * skb_frag_address_safe - gets the address of the data contained in a paged fragment
 * @frag: the paged fragment buffer
 *
 * Returns: the address of the data within @frag. Checks that the page
 * is mapped and returns %NULL otherwise.
 */
static inline void *skb_frag_address_safe(const skb_frag_t *frag)
{
        void *ptr = page_address(skb_frag_page(frag));
        if (unlikely(!ptr))
                return NULL;

        return ptr + skb_frag_off(frag);
}

/**
 * skb_frag_page_copy() - sets the page in a fragment from another fragment
 * @fragto: skb fragment where page is set
 * @fragfrom: skb fragment page is copied from
 */
static inline void skb_frag_page_copy(skb_frag_t *fragto,
                                      const skb_frag_t *fragfrom)
{
        fragto->netmem = fragfrom->netmem;
}

bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio);

/**
 * __skb_frag_dma_map - maps a paged fragment via the DMA API
 * @dev: the device to map the fragment to
 * @frag: the paged fragment to map
 * @offset: the offset within the fragment (starting at the
 *          fragment's own offset)
 * @size: the number of bytes to map
 * @dir: the direction of the mapping (``PCI_DMA_*``)
 *
 * Maps the page associated with @frag to @device.
 */
static inline dma_addr_t __skb_frag_dma_map(struct device *dev,
                                            const skb_frag_t *frag,
                                            size_t offset, size_t size,
                                            enum dma_data_direction dir)
{
        return dma_map_page(dev, skb_frag_page(frag),
                            skb_frag_off(frag) + offset, size, dir);
}

#define skb_frag_dma_map(dev, frag, ...)                                \
        CONCATENATE(_skb_frag_dma_map,                                        \
                    COUNT_ARGS(__VA_ARGS__))(dev, frag, ##__VA_ARGS__)

#define __skb_frag_dma_map1(dev, frag, offset, uf, uo) ({                \
        const skb_frag_t *uf = (frag);                                        \
        size_t uo = (offset);                                                \
                                                                        \
        __skb_frag_dma_map(dev, uf, uo, skb_frag_size(uf) - uo,                \
                           DMA_TO_DEVICE);                                \
})
#define _skb_frag_dma_map1(dev, frag, offset)                                \
        __skb_frag_dma_map1(dev, frag, offset, __UNIQUE_ID(frag_),        \
                            __UNIQUE_ID(offset_))
#define _skb_frag_dma_map0(dev, frag)                                        \
        _skb_frag_dma_map1(dev, frag, 0)
#define _skb_frag_dma_map2(dev, frag, offset, size)                        \
        __skb_frag_dma_map(dev, frag, offset, size, DMA_TO_DEVICE)
#define _skb_frag_dma_map3(dev, frag, offset, size, dir)                \
        __skb_frag_dma_map(dev, frag, offset, size, dir)

static inline struct sk_buff *pskb_copy(struct sk_buff *skb,
                                        gfp_t gfp_mask)
{
        return __pskb_copy(skb, skb_headroom(skb), gfp_mask);
}


static inline struct sk_buff *pskb_copy_for_clone(struct sk_buff *skb,
                                                  gfp_t gfp_mask)
{
        return __pskb_copy_fclone(skb, skb_headroom(skb), gfp_mask, true);
}


/**
 *        skb_clone_writable - is the header of a clone writable
 *        @skb: buffer to check
 *        @len: length up to which to write
 *
 *        Returns true if modifying the header part of the cloned buffer
 *        does not requires the data to be copied.
 */
static inline int skb_clone_writable(const struct sk_buff *skb, unsigned int len)
{
        return !skb_header_cloned(skb) &&
               skb_headroom(skb) + len <= skb->hdr_len;
}

static inline int skb_try_make_writable(struct sk_buff *skb,
                                        unsigned int write_len)
{
        return skb_cloned(skb) && !skb_clone_writable(skb, write_len) &&
               pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
}

static inline int __skb_cow(struct sk_buff *skb, unsigned int headroom,
                            int cloned)
{
        int delta = 0;

        if (headroom > skb_headroom(skb))
                delta = headroom - skb_headroom(skb);

        if (delta || cloned)
                return pskb_expand_head(skb, ALIGN(delta, NET_SKB_PAD), 0,
                                        GFP_ATOMIC);
        return 0;
}

/**
 *        skb_cow - copy header of skb when it is required
 *        @skb: buffer to cow
 *        @headroom: needed headroom
 *
 *        If the skb passed lacks sufficient headroom or its data part
 *        is shared, data is reallocated. If reallocation fails, an error
 *        is returned and original skb is not changed.
 *
 *        The result is skb with writable area skb->head...skb->tail
 *        and at least @headroom of space at head.
 */
static inline int skb_cow(struct sk_buff *skb, unsigned int headroom)
{
        return __skb_cow(skb, headroom, skb_cloned(skb));
}

/**
 *        skb_cow_head - skb_cow but only making the head writable
 *        @skb: buffer to cow
 *        @headroom: needed headroom
 *
 *        This function is identical to skb_cow except that we replace the
 *        skb_cloned check by skb_header_cloned.  It should be used when
 *        you only need to push on some header and do not need to modify
 *        the data.
 */
static inline int skb_cow_head(struct sk_buff *skb, unsigned int headroom)
{
        return __skb_cow(skb, headroom, skb_header_cloned(skb));
}

/**
 *        skb_padto        - pad an skbuff up to a minimal size
 *        @skb: buffer to pad
 *        @len: minimal length
 *
 *        Pads up a buffer to ensure the trailing bytes exist and are
 *        blanked. If the buffer already contains sufficient data it
 *        is untouched. Otherwise it is extended. Returns zero on
 *        success. The skb is freed on error.
 */
static inline int skb_padto(struct sk_buff *skb, unsigned int len)
{
        unsigned int size = skb->len;
        if (likely(size >= len))
                return 0;
        return skb_pad(skb, len - size);
}

/**
 *        __skb_put_padto - increase size and pad an skbuff up to a minimal size
 *        @skb: buffer to pad
 *        @len: minimal length
 *        @free_on_error: free buffer on error
 *
 *        Pads up a buffer to ensure the trailing bytes exist and are
 *        blanked. If the buffer already contains sufficient data it
 *        is untouched. Otherwise it is extended. Returns zero on
 *        success. The skb is freed on error if @free_on_error is true.
 */
static inline int __must_check __skb_put_padto(struct sk_buff *skb,
                                               unsigned int len,
                                               bool free_on_error)
{
        unsigned int size = skb->len;

        if (unlikely(size < len)) {
                len -= size;
                if (__skb_pad(skb, len, free_on_error))
                        return -ENOMEM;
                __skb_put(skb, len);
        }
        return 0;
}

/**
 *        skb_put_padto - increase size and pad an skbuff up to a minimal size
 *        @skb: buffer to pad
 *        @len: minimal length
 *
 *        Pads up a buffer to ensure the trailing bytes exist and are
 *        blanked. If the buffer already contains sufficient data it
 *        is untouched. Otherwise it is extended. Returns zero on
 *        success. The skb is freed on error.
 */
static inline int __must_check skb_put_padto(struct sk_buff *skb, unsigned int len)
{
        return __skb_put_padto(skb, len, true);
}

bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i)
        __must_check;

static inline bool skb_can_coalesce(struct sk_buff *skb, int i,
                                    const struct page *page, int off)
{
        if (skb_zcopy(skb))
                return false;
        if (i) {
                const skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];

                return page == skb_frag_page(frag) &&
                       off == skb_frag_off(frag) + skb_frag_size(frag);
        }
        return false;
}

static inline int __skb_linearize(struct sk_buff *skb)
{
        return __pskb_pull_tail(skb, skb->data_len) ? 0 : -ENOMEM;
}

/**
 *        skb_linearize - convert paged skb to linear one
 *        @skb: buffer to linarize
 *
 *        If there is no free memory -ENOMEM is returned, otherwise zero
 *        is returned and the old skb data released.
 */
static inline int skb_linearize(struct sk_buff *skb)
{
        return skb_is_nonlinear(skb) ? __skb_linearize(skb) : 0;
}

/**
 * skb_has_shared_frag - can any frag be overwritten
 * @skb: buffer to test
 *
 * Return: true if the skb has at least one frag that might be modified
 * by an external entity (as in vmsplice()/sendfile())
 */
static inline bool skb_has_shared_frag(const struct sk_buff *skb)
{
        return skb_is_nonlinear(skb) &&
               skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG;
}

/**
 *        skb_linearize_cow - make sure skb is linear and writable
 *        @skb: buffer to process
 *
 *        If there is no free memory -ENOMEM is returned, otherwise zero
 *        is returned and the old skb data released.
 */
static inline int skb_linearize_cow(struct sk_buff *skb)
{
        return skb_is_nonlinear(skb) || skb_cloned(skb) ?
               __skb_linearize(skb) : 0;
}

static __always_inline void
__skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len,
                     unsigned int off)
{
        if (skb->ip_summed == CHECKSUM_COMPLETE)
                skb->csum = csum_block_sub(skb->csum,
                                           csum_partial(start, len, 0), off);
        else if (skb->ip_summed == CHECKSUM_PARTIAL &&
                 skb_checksum_start_offset(skb) < 0)
                skb->ip_summed = CHECKSUM_NONE;
}

/**
 *        skb_postpull_rcsum - update checksum for received skb after pull
 *        @skb: buffer to update
 *        @start: start of data before pull
 *        @len: length of data pulled
 *
 *        After doing a pull on a received packet, you need to call this to
 *        update the CHECKSUM_COMPLETE checksum, or set ip_summed to
 *        CHECKSUM_NONE so that it can be recomputed from scratch.
 */
static inline void skb_postpull_rcsum(struct sk_buff *skb,
                                      const void *start, unsigned int len)
{
        if (skb->ip_summed == CHECKSUM_COMPLETE)
                skb->csum = wsum_negate(csum_partial(start, len,
                                                     wsum_negate(skb->csum)));
        else if (skb->ip_summed == CHECKSUM_PARTIAL &&
                 skb_checksum_start_offset(skb) < 0)
                skb->ip_summed = CHECKSUM_NONE;
}

static __always_inline void
__skb_postpush_rcsum(struct sk_buff *skb, const void *start, unsigned int len,
                     unsigned int off)
{
        if (skb->ip_summed == CHECKSUM_COMPLETE)
                skb->csum = csum_block_add(skb->csum,
                                           csum_partial(start, len, 0), off);
}

/**
 *        skb_postpush_rcsum - update checksum for received skb after push
 *        @skb: buffer to update
 *        @start: start of data after push
 *        @len: length of data pushed
 *
 *        After doing a push on a received packet, you need to call this to
 *        update the CHECKSUM_COMPLETE checksum.
 */
static inline void skb_postpush_rcsum(struct sk_buff *skb,
                                      const void *start, unsigned int len)
{
        __skb_postpush_rcsum(skb, start, len, 0);
}

void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len);

/**
 *        skb_push_rcsum - push skb and update receive checksum
 *        @skb: buffer to update
 *        @len: length of data pulled
 *
 *        This function performs an skb_push on the packet and updates
 *        the CHECKSUM_COMPLETE checksum.  It should be used on
 *        receive path processing instead of skb_push unless you know
 *        that the checksum difference is zero (e.g., a valid IP header)
 *        or you are setting ip_summed to CHECKSUM_NONE.
 */
static inline void *skb_push_rcsum(struct sk_buff *skb, unsigned int len)
{
        skb_push(skb, len);
        skb_postpush_rcsum(skb, skb->data, len);
        return skb->data;
}

int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len);
/**
 *        pskb_trim_rcsum - trim received skb and update checksum
 *        @skb: buffer to trim
 *        @len: new length
 *
 *        This is exactly the same as pskb_trim except that it ensures the
 *        checksum of received packets are still valid after the operation.
 *        It can change skb pointers.
 */

static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len)
{
        skb_might_realloc(skb);
        if (likely(len >= skb->len))
                return 0;
        return pskb_trim_rcsum_slow(skb, len);
}

static inline int __skb_trim_rcsum(struct sk_buff *skb, unsigned int len)
{
        if (skb->ip_summed == CHECKSUM_COMPLETE)
                skb->ip_summed = CHECKSUM_NONE;
        __skb_trim(skb, len);
        return 0;
}

static inline int __skb_grow_rcsum(struct sk_buff *skb, unsigned int len)
{
        if (skb->ip_summed == CHECKSUM_COMPLETE)
                skb->ip_summed = CHECKSUM_NONE;
        return __skb_grow(skb, len);
}

#define rb_to_skb(rb) rb_entry_safe(rb, struct sk_buff, rbnode)
#define skb_rb_first(root) rb_to_skb(rb_first(root))
#define skb_rb_last(root)  rb_to_skb(rb_last(root))
#define skb_rb_next(skb)   rb_to_skb(rb_next(&(skb)->rbnode))
#define skb_rb_prev(skb)   rb_to_skb(rb_prev(&(skb)->rbnode))

#define skb_queue_walk(queue, skb) \
                for (skb = (queue)->next;                                        \
                     skb != (struct sk_buff *)(queue);                                \
                     skb = skb->next)

#define skb_queue_walk_safe(queue, skb, tmp)                                        \
                for (skb = (queue)->next, tmp = skb->next;                        \
                     skb != (struct sk_buff *)(queue);                                \
                     skb = tmp, tmp = skb->next)

#define skb_queue_walk_from(queue, skb)                                                \
                for (; skb != (struct sk_buff *)(queue);                        \
                     skb = skb->next)

#define skb_rbtree_walk(skb, root)                                                \
                for (skb = skb_rb_first(root); skb != NULL;                        \
                     skb = skb_rb_next(skb))

#define skb_rbtree_walk_from(skb)                                                \
                for (; skb != NULL;                                                \
                     skb = skb_rb_next(skb))

#define skb_rbtree_walk_from_safe(skb, tmp)                                        \
                for (; tmp = skb ? skb_rb_next(skb) : NULL, (skb != NULL);        \
                     skb = tmp)

#define skb_queue_walk_from_safe(queue, skb, tmp)                                \
                for (tmp = skb->next;                                                \
                     skb != (struct sk_buff *)(queue);                                \
                     skb = tmp, tmp = skb->next)

#define skb_queue_reverse_walk(queue, skb) \
                for (skb = (queue)->prev;                                        \
                     skb != (struct sk_buff *)(queue);                                \
                     skb = skb->prev)

#define skb_queue_reverse_walk_safe(queue, skb, tmp)                                \
                for (skb = (queue)->prev, tmp = skb->prev;                        \
                     skb != (struct sk_buff *)(queue);                                \
                     skb = tmp, tmp = skb->prev)

#define skb_queue_reverse_walk_from_safe(queue, skb, tmp)                        \
                for (tmp = skb->prev;                                                \
                     skb != (struct sk_buff *)(queue);                                \
                     skb = tmp, tmp = skb->prev)

static inline bool skb_has_frag_list(const struct sk_buff *skb)
{
        return skb_shinfo(skb)->frag_list != NULL;
}

static inline void skb_frag_list_init(struct sk_buff *skb)
{
        skb_shinfo(skb)->frag_list = NULL;
}

#define skb_walk_frags(skb, iter)        \
        for (iter = skb_shinfo(skb)->frag_list; iter; iter = iter->next)


int __skb_wait_for_more_packets(struct sock *sk, struct sk_buff_head *queue,
                                int *err, long *timeo_p,
                                const struct sk_buff *skb);
struct sk_buff *__skb_try_recv_from_queue(struct sock *sk,
                                          struct sk_buff_head *queue,
                                          unsigned int flags,
                                          int *off, int *err,
                                          struct sk_buff **last);
struct sk_buff *__skb_try_recv_datagram(struct sock *sk,
                                        struct sk_buff_head *queue,
                                        unsigned int flags, int *off, int *err,
                                        struct sk_buff **last);
struct sk_buff *__skb_recv_datagram(struct sock *sk,
                                    struct sk_buff_head *sk_queue,
                                    unsigned int flags, int *off, int *err);
struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned int flags, int *err);
__poll_t datagram_poll(struct file *file, struct socket *sock,
                           struct poll_table_struct *wait);
int skb_copy_datagram_iter(const struct sk_buff *from, int offset,
                           struct iov_iter *to, int size);
static inline int skb_copy_datagram_msg(const struct sk_buff *from, int offset,
                                        struct msghdr *msg, int size)
{
        return skb_copy_datagram_iter(from, offset, &msg->msg_iter, size);
}
int skb_copy_and_csum_datagram_msg(struct sk_buff *skb, int hlen,
                                   struct msghdr *msg);
int skb_copy_and_hash_datagram_iter(const struct sk_buff *skb, int offset,
                           struct iov_iter *to, int len,
                           struct ahash_request *hash);
int skb_copy_datagram_from_iter(struct sk_buff *skb, int offset,
                                 struct iov_iter *from, int len);
int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *frm);
void skb_free_datagram(struct sock *sk, struct sk_buff *skb);
int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags);
int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len);
int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len);
__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, u8 *to,
                              int len);
int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
                    struct pipe_inode_info *pipe, unsigned int len,
                    unsigned int flags);
int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
                         int len);
int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len);
void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);
unsigned int skb_zerocopy_headlen(const struct sk_buff *from);
int skb_zerocopy(struct sk_buff *to, struct sk_buff *from,
                 int len, int hlen);
void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len);
int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen);
void skb_scrub_packet(struct sk_buff *skb, bool xnet);
struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features);
struct sk_buff *skb_segment_list(struct sk_buff *skb, netdev_features_t features,
                                 unsigned int offset);
struct sk_buff *skb_vlan_untag(struct sk_buff *skb);
int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len);
int skb_ensure_writable_head_tail(struct sk_buff *skb, struct net_device *dev);
int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci);
int skb_vlan_pop(struct sk_buff *skb);
int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci);
int skb_eth_pop(struct sk_buff *skb);
int skb_eth_push(struct sk_buff *skb, const unsigned char *dst,
                 const unsigned char *src);
int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
                  int mac_len, bool ethernet);
int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
                 bool ethernet);
int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse);
int skb_mpls_dec_ttl(struct sk_buff *skb);
struct sk_buff *pskb_extract(struct sk_buff *skb, int off, int to_copy,
                             gfp_t gfp);

static inline int memcpy_from_msg(void *data, struct msghdr *msg, int len)
{
        return copy_from_iter_full(data, len, &msg->msg_iter) ? 0 : -EFAULT;
}

static inline int memcpy_to_msg(struct msghdr *msg, void *data, int len)
{
        return copy_to_iter(data, len, &msg->msg_iter) == len ? 0 : -EFAULT;
}

struct skb_checksum_ops {
        __wsum (*update)(const void *mem, int len, __wsum wsum);
        __wsum (*combine)(__wsum csum, __wsum csum2, int offset, int len);
};

extern const struct skb_checksum_ops *crc32c_csum_stub __read_mostly;

__wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
                      __wsum csum, const struct skb_checksum_ops *ops);
__wsum skb_checksum(const struct sk_buff *skb, int offset, int len,
                    __wsum csum);

static inline void * __must_check
__skb_header_pointer(const struct sk_buff *skb, int offset, int len,
                     const void *data, int hlen, void *buffer)
{
        if (likely(hlen - offset >= len))
                return (void *)data + offset;

        if (!skb || unlikely(skb_copy_bits(skb, offset, buffer, len) < 0))
                return NULL;

        return buffer;
}

static inline void * __must_check
skb_header_pointer(const struct sk_buff *skb, int offset, int len, void *buffer)
{
        return __skb_header_pointer(skb, offset, len, skb->data,
                                    skb_headlen(skb), buffer);
}

static inline void * __must_check
skb_pointer_if_linear(const struct sk_buff *skb, int offset, int len)
{
        if (likely(skb_headlen(skb) - offset >= len))
                return skb->data + offset;
        return NULL;
}

/**
 *        skb_needs_linearize - check if we need to linearize a given skb
 *                              depending on the given device features.
 *        @skb: socket buffer to check
 *        @features: net device features
 *
 *        Returns true if either:
 *        1. skb has frag_list and the device doesn't support FRAGLIST, or
 *        2. skb is fragmented and the device does not support SG.
 */
static inline bool skb_needs_linearize(struct sk_buff *skb,
                                       netdev_features_t features)
{
        return skb_is_nonlinear(skb) &&
               ((skb_has_frag_list(skb) && !(features & NETIF_F_FRAGLIST)) ||
                (skb_shinfo(skb)->nr_frags && !(features & NETIF_F_SG)));
}

static inline void skb_copy_from_linear_data(const struct sk_buff *skb,
                                             void *to,
                                             const unsigned int len)
{
        memcpy(to, skb->data, len);
}

static inline void skb_copy_from_linear_data_offset(const struct sk_buff *skb,
                                                    const int offset, void *to,
                                                    const unsigned int len)
{
        memcpy(to, skb->data + offset, len);
}

static inline void skb_copy_to_linear_data(struct sk_buff *skb,
                                           const void *from,
                                           const unsigned int len)
{
        memcpy(skb->data, from, len);
}

static inline void skb_copy_to_linear_data_offset(struct sk_buff *skb,
                                                  const int offset,
                                                  const void *from,
                                                  const unsigned int len)
{
        memcpy(skb->data + offset, from, len);
}

void skb_init(void);

static inline ktime_t skb_get_ktime(const struct sk_buff *skb)
{
        return skb->tstamp;
}

/**
 *        skb_get_timestamp - get timestamp from a skb
 *        @skb: skb to get stamp from
 *        @stamp: pointer to struct __kernel_old_timeval to store stamp in
 *
 *        Timestamps are stored in the skb as offsets to a base timestamp.
 *        This function converts the offset back to a struct timeval and stores
 *        it in stamp.
 */
static inline void skb_get_timestamp(const struct sk_buff *skb,
                                     struct __kernel_old_timeval *stamp)
{
        *stamp = ns_to_kernel_old_timeval(skb->tstamp);
}

static inline void skb_get_new_timestamp(const struct sk_buff *skb,
                                         struct __kernel_sock_timeval *stamp)
{
        struct timespec64 ts = ktime_to_timespec64(skb->tstamp);

        stamp->tv_sec = ts.tv_sec;
        stamp->tv_usec = ts.tv_nsec / 1000;
}

static inline void skb_get_timestampns(const struct sk_buff *skb,
                                       struct __kernel_old_timespec *stamp)
{
        struct timespec64 ts = ktime_to_timespec64(skb->tstamp);

        stamp->tv_sec = ts.tv_sec;
        stamp->tv_nsec = ts.tv_nsec;
}

static inline void skb_get_new_timestampns(const struct sk_buff *skb,
                                           struct __kernel_timespec *stamp)
{
        struct timespec64 ts = ktime_to_timespec64(skb->tstamp);

        stamp->tv_sec = ts.tv_sec;
        stamp->tv_nsec = ts.tv_nsec;
}

static inline void __net_timestamp(struct sk_buff *skb)
{
        skb->tstamp = ktime_get_real();
        skb->tstamp_type = SKB_CLOCK_REALTIME;
}

static inline ktime_t net_timedelta(ktime_t t)
{
        return ktime_sub(ktime_get_real(), t);
}

static inline void skb_set_delivery_time(struct sk_buff *skb, ktime_t kt,
                                         u8 tstamp_type)
{
        skb->tstamp = kt;

        if (kt)
                skb->tstamp_type = tstamp_type;
        else
                skb->tstamp_type = SKB_CLOCK_REALTIME;
}

static inline void skb_set_delivery_type_by_clockid(struct sk_buff *skb,
                                                    ktime_t kt, clockid_t clockid)
{
        u8 tstamp_type = SKB_CLOCK_REALTIME;

        switch (clockid) {
        case CLOCK_REALTIME:
                break;
        case CLOCK_MONOTONIC:
                tstamp_type = SKB_CLOCK_MONOTONIC;
                break;
        case CLOCK_TAI:
                tstamp_type = SKB_CLOCK_TAI;
                break;
        default:
                WARN_ON_ONCE(1);
                kt = 0;
        }

        skb_set_delivery_time(skb, kt, tstamp_type);
}

DECLARE_STATIC_KEY_FALSE(netstamp_needed_key);

/* It is used in the ingress path to clear the delivery_time.
 * If needed, set the skb->tstamp to the (rcv) timestamp.
 */
static inline void skb_clear_delivery_time(struct sk_buff *skb)
{
        if (skb->tstamp_type) {
                skb->tstamp_type = SKB_CLOCK_REALTIME;
                if (static_branch_unlikely(&netstamp_needed_key))
                        skb->tstamp = ktime_get_real();
                else
                        skb->tstamp = 0;
        }
}

static inline void skb_clear_tstamp(struct sk_buff *skb)
{
        if (skb->tstamp_type)
                return;

        skb->tstamp = 0;
}

static inline ktime_t skb_tstamp(const struct sk_buff *skb)
{
        if (skb->tstamp_type)
                return 0;

        return skb->tstamp;
}

static inline ktime_t skb_tstamp_cond(const struct sk_buff *skb, bool cond)
{
        if (skb->tstamp_type != SKB_CLOCK_MONOTONIC && skb->tstamp)
                return skb->tstamp;

        if (static_branch_unlikely(&netstamp_needed_key) || cond)
                return ktime_get_real();

        return 0;
}

static inline u8 skb_metadata_len(const struct sk_buff *skb)
{
        return skb_shinfo(skb)->meta_len;
}

static inline void *skb_metadata_end(const struct sk_buff *skb)
{
        return skb_mac_header(skb);
}

static inline bool __skb_metadata_differs(const struct sk_buff *skb_a,
                                          const struct sk_buff *skb_b,
                                          u8 meta_len)
{
        const void *a = skb_metadata_end(skb_a);
        const void *b = skb_metadata_end(skb_b);
        u64 diffs = 0;

        if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
            BITS_PER_LONG != 64)
                goto slow;

        /* Using more efficient variant than plain call to memcmp(). */
        switch (meta_len) {
#define __it(x, op) (x -= sizeof(u##op))
#define __it_diff(a, b, op) (*(u##op *)__it(a, op)) ^ (*(u##op *)__it(b, op))
        case 32: diffs |= __it_diff(a, b, 64);
                fallthrough;
        case 24: diffs |= __it_diff(a, b, 64);
                fallthrough;
        case 16: diffs |= __it_diff(a, b, 64);
                fallthrough;
        case  8: diffs |= __it_diff(a, b, 64);
                break;
        case 28: diffs |= __it_diff(a, b, 64);
                fallthrough;
        case 20: diffs |= __it_diff(a, b, 64);
                fallthrough;
        case 12: diffs |= __it_diff(a, b, 64);
                fallthrough;
        case  4: diffs |= __it_diff(a, b, 32);
                break;
        default:
slow:
                return memcmp(a - meta_len, b - meta_len, meta_len);
        }
        return diffs;
}

static inline bool skb_metadata_differs(const struct sk_buff *skb_a,
                                        const struct sk_buff *skb_b)
{
        u8 len_a = skb_metadata_len(skb_a);
        u8 len_b = skb_metadata_len(skb_b);

        if (!(len_a | len_b))
                return false;

        return len_a != len_b ?
               true : __skb_metadata_differs(skb_a, skb_b, len_a);
}

static inline void skb_metadata_set(struct sk_buff *skb, u8 meta_len)
{
        skb_shinfo(skb)->meta_len = meta_len;
}

static inline void skb_metadata_clear(struct sk_buff *skb)
{
        skb_metadata_set(skb, 0);
}

struct sk_buff *skb_clone_sk(struct sk_buff *skb);

#ifdef CONFIG_NETWORK_PHY_TIMESTAMPING

void skb_clone_tx_timestamp(struct sk_buff *skb);
bool skb_defer_rx_timestamp(struct sk_buff *skb);

#else /* CONFIG_NETWORK_PHY_TIMESTAMPING */

static inline void skb_clone_tx_timestamp(struct sk_buff *skb)
{
}

static inline bool skb_defer_rx_timestamp(struct sk_buff *skb)
{
        return false;
}

#endif /* !CONFIG_NETWORK_PHY_TIMESTAMPING */

/**
 * skb_complete_tx_timestamp() - deliver cloned skb with tx timestamps
 *
 * PHY drivers may accept clones of transmitted packets for
 * timestamping via their phy_driver.txtstamp method. These drivers
 * must call this function to return the skb back to the stack with a
 * timestamp.
 *
 * @skb: clone of the original outgoing packet
 * @hwtstamps: hardware time stamps
 *
 */
void skb_complete_tx_timestamp(struct sk_buff *skb,
                               struct skb_shared_hwtstamps *hwtstamps);

void __skb_tstamp_tx(struct sk_buff *orig_skb, const struct sk_buff *ack_skb,
                     struct skb_shared_hwtstamps *hwtstamps,
                     struct sock *sk, int tstype);

/**
 * skb_tstamp_tx - queue clone of skb with send time stamps
 * @orig_skb:        the original outgoing packet
 * @hwtstamps:        hardware time stamps, may be NULL if not available
 *
 * If the skb has a socket associated, then this function clones the
 * skb (thus sharing the actual data and optional structures), stores
 * the optional hardware time stamping information (if non NULL) or
 * generates a software time stamp (otherwise), then queues the clone
 * to the error queue of the socket.  Errors are silently ignored.
 */
void skb_tstamp_tx(struct sk_buff *orig_skb,
                   struct skb_shared_hwtstamps *hwtstamps);

/**
 * skb_tx_timestamp() - Driver hook for transmit timestamping
 *
 * Ethernet MAC Drivers should call this function in their hard_xmit()
 * function immediately before giving the sk_buff to the MAC hardware.
 *
 * Specifically, one should make absolutely sure that this function is
 * called before TX completion of this packet can trigger.  Otherwise
 * the packet could potentially already be freed.
 *
 * @skb: A socket buffer.
 */
static inline void skb_tx_timestamp(struct sk_buff *skb)
{
        skb_clone_tx_timestamp(skb);
        if (skb_shinfo(skb)->tx_flags & (SKBTX_SW_TSTAMP | SKBTX_BPF))
                skb_tstamp_tx(skb, NULL);
}

/**
 * skb_complete_wifi_ack - deliver skb with wifi status
 *
 * @skb: the original outgoing packet
 * @acked: ack status
 *
 */
void skb_complete_wifi_ack(struct sk_buff *skb, bool acked);

__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len);
__sum16 __skb_checksum_complete(struct sk_buff *skb);

static inline int skb_csum_unnecessary(const struct sk_buff *skb)
{
        return ((skb->ip_summed == CHECKSUM_UNNECESSARY) ||
                skb->csum_valid ||
                (skb->ip_summed == CHECKSUM_PARTIAL &&
                 skb_checksum_start_offset(skb) >= 0));
}

/**
 *        skb_checksum_complete - Calculate checksum of an entire packet
 *        @skb: packet to process
 *
 *        This function calculates the checksum over the entire packet plus
 *        the value of skb->csum.  The latter can be used to supply the
 *        checksum of a pseudo header as used by TCP/UDP.  It returns the
 *        checksum.
 *
 *        For protocols that contain complete checksums such as ICMP/TCP/UDP,
 *        this function can be used to verify that checksum on received
 *        packets.  In that case the function should return zero if the
 *        checksum is correct.  In particular, this function will return zero
 *        if skb->ip_summed is CHECKSUM_UNNECESSARY which indicates that the
 *        hardware has already verified the correctness of the checksum.
 */
static inline __sum16 skb_checksum_complete(struct sk_buff *skb)
{
        return skb_csum_unnecessary(skb) ?
               0 : __skb_checksum_complete(skb);
}

static inline void __skb_decr_checksum_unnecessary(struct sk_buff *skb)
{
        if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
                if (skb->csum_level == 0)
                        skb->ip_summed = CHECKSUM_NONE;
                else
                        skb->csum_level--;
        }
}

static inline void __skb_incr_checksum_unnecessary(struct sk_buff *skb)
{
        if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
                if (skb->csum_level < SKB_MAX_CSUM_LEVEL)
                        skb->csum_level++;
        } else if (skb->ip_summed == CHECKSUM_NONE) {
                skb->ip_summed = CHECKSUM_UNNECESSARY;
                skb->csum_level = 0;
        }
}

static inline void __skb_reset_checksum_unnecessary(struct sk_buff *skb)
{
        if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
                skb->ip_summed = CHECKSUM_NONE;
                skb->csum_level = 0;
        }
}

/* Check if we need to perform checksum complete validation.
 *
 * Returns: true if checksum complete is needed, false otherwise
 * (either checksum is unnecessary or zero checksum is allowed).
 */
static inline bool __skb_checksum_validate_needed(struct sk_buff *skb,
                                                  bool zero_okay,
                                                  __sum16 check)
{
        if (skb_csum_unnecessary(skb) || (zero_okay && !check)) {
                skb->csum_valid = 1;
                __skb_decr_checksum_unnecessary(skb);
                return false;
        }

        return true;
}

/* For small packets <= CHECKSUM_BREAK perform checksum complete directly
 * in checksum_init.
 */
#define CHECKSUM_BREAK 76

/* Unset checksum-complete
 *
 * Unset checksum complete can be done when packet is being modified
 * (uncompressed for instance) and checksum-complete value is
 * invalidated.
 */
static inline void skb_checksum_complete_unset(struct sk_buff *skb)
{
        if (skb->ip_summed == CHECKSUM_COMPLETE)
                skb->ip_summed = CHECKSUM_NONE;
}

/* Validate (init) checksum based on checksum complete.
 *
 * Return values:
 *   0: checksum is validated or try to in skb_checksum_complete. In the latter
 *        case the ip_summed will not be CHECKSUM_UNNECESSARY and the pseudo
 *        checksum is stored in skb->csum for use in __skb_checksum_complete
 *   non-zero: value of invalid checksum
 *
 */
static inline __sum16 __skb_checksum_validate_complete(struct sk_buff *skb,
                                                       bool complete,
                                                       __wsum psum)
{
        if (skb->ip_summed == CHECKSUM_COMPLETE) {
                if (!csum_fold(csum_add(psum, skb->csum))) {
                        skb->csum_valid = 1;
                        return 0;
                }
        }

        skb->csum = psum;

        if (complete || skb->len <= CHECKSUM_BREAK) {
                __sum16 csum;

                csum = __skb_checksum_complete(skb);
                skb->csum_valid = !csum;
                return csum;
        }

        return 0;
}

static inline __wsum null_compute_pseudo(struct sk_buff *skb, int proto)
{
        return 0;
}

/* Perform checksum validate (init). Note that this is a macro since we only
 * want to calculate the pseudo header which is an input function if necessary.
 * First we try to validate without any computation (checksum unnecessary) and
 * then calculate based on checksum complete calling the function to compute
 * pseudo header.
 *
 * Return values:
 *   0: checksum is validated or try to in skb_checksum_complete
 *   non-zero: value of invalid checksum
 */
#define __skb_checksum_validate(skb, proto, complete,                        \
                                zero_okay, check, compute_pseudo)        \
({                                                                        \
        __sum16 __ret = 0;                                                \
        skb->csum_valid = 0;                                                \
        if (__skb_checksum_validate_needed(skb, zero_okay, check))        \
                __ret = __skb_checksum_validate_complete(skb,                \
                                complete, compute_pseudo(skb, proto));        \
        __ret;                                                                \
})

#define skb_checksum_init(skb, proto, compute_pseudo)                        \
        __skb_checksum_validate(skb, proto, false, false, 0, compute_pseudo)

#define skb_checksum_init_zero_check(skb, proto, check, compute_pseudo)        \
        __skb_checksum_validate(skb, proto, false, true, check, compute_pseudo)

#define skb_checksum_validate(skb, proto, compute_pseudo)                \
        __skb_checksum_validate(skb, proto, true, false, 0, compute_pseudo)

#define skb_checksum_validate_zero_check(skb, proto, check,                \
                                         compute_pseudo)                \
        __skb_checksum_validate(skb, proto, true, true, check, compute_pseudo)

#define skb_checksum_simple_validate(skb)                                \
        __skb_checksum_validate(skb, 0, true, false, 0, null_compute_pseudo)

static inline bool __skb_checksum_convert_check(struct sk_buff *skb)
{
        return (skb->ip_summed == CHECKSUM_NONE && skb->csum_valid);
}

static inline void __skb_checksum_convert(struct sk_buff *skb, __wsum pseudo)
{
        skb->csum = ~pseudo;
        skb->ip_summed = CHECKSUM_COMPLETE;
}

#define skb_checksum_try_convert(skb, proto, compute_pseudo)        \
do {                                                                        \
        if (__skb_checksum_convert_check(skb))                                \
                __skb_checksum_convert(skb, compute_pseudo(skb, proto)); \
} while (0)

static inline void skb_remcsum_adjust_partial(struct sk_buff *skb, void *ptr,
                                              u16 start, u16 offset)
{
        skb->ip_summed = CHECKSUM_PARTIAL;
        skb->csum_start = ((unsigned char *)ptr + start) - skb->head;
        skb->csum_offset = offset - start;
}

/* Update skbuf and packet to reflect the remote checksum offload operation.
 * When called, ptr indicates the starting point for skb->csum when
 * ip_summed is CHECKSUM_COMPLETE. If we need create checksum complete
 * here, skb_postpull_rcsum is done so skb->csum start is ptr.
 */
static inline void skb_remcsum_process(struct sk_buff *skb, void *ptr,
                                       int start, int offset, bool nopartial)
{
        __wsum delta;

        if (!nopartial) {
                skb_remcsum_adjust_partial(skb, ptr, start, offset);
                return;
        }

        if (unlikely(skb->ip_summed != CHECKSUM_COMPLETE)) {
                __skb_checksum_complete(skb);
                skb_postpull_rcsum(skb, skb->data, ptr - (void *)skb->data);
        }

        delta = remcsum_adjust(ptr, skb->csum, start, offset);

        /* Adjust skb->csum since we changed the packet */
        skb->csum = csum_add(skb->csum, delta);
}

static inline struct nf_conntrack *skb_nfct(const struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
        return (void *)(skb->_nfct & NFCT_PTRMASK);
#else
        return NULL;
#endif
}

static inline unsigned long skb_get_nfct(const struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
        return skb->_nfct;
#else
        return 0UL;
#endif
}

static inline void skb_set_nfct(struct sk_buff *skb, unsigned long nfct)
{
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
        skb->slow_gro |= !!nfct;
        skb->_nfct = nfct;
#endif
}

#ifdef CONFIG_SKB_EXTENSIONS
enum skb_ext_id {
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
        SKB_EXT_BRIDGE_NF,
#endif
#ifdef CONFIG_XFRM
        SKB_EXT_SEC_PATH,
#endif
#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
        TC_SKB_EXT,
#endif
#if IS_ENABLED(CONFIG_MPTCP)
        SKB_EXT_MPTCP,
#endif
#if IS_ENABLED(CONFIG_MCTP_FLOWS)
        SKB_EXT_MCTP,
#endif
        SKB_EXT_NUM, /* must be last */
};

/**
 *        struct skb_ext - sk_buff extensions
 *        @refcnt: 1 on allocation, deallocated on 0
 *        @offset: offset to add to @data to obtain extension address
 *        @chunks: size currently allocated, stored in SKB_EXT_ALIGN_SHIFT units
 *        @data: start of extension data, variable sized
 *
 *        Note: offsets/lengths are stored in chunks of 8 bytes, this allows
 *        to use 'u8' types while allowing up to 2kb worth of extension data.
 */
struct skb_ext {
        refcount_t refcnt;
        u8 offset[SKB_EXT_NUM]; /* in chunks of 8 bytes */
        u8 chunks;                /* same */
        char data[] __aligned(8);
};

struct skb_ext *__skb_ext_alloc(gfp_t flags);
void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id,
                    struct skb_ext *ext);
void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id);
void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id);
void __skb_ext_put(struct skb_ext *ext);

static inline void skb_ext_put(struct sk_buff *skb)
{
        if (skb->active_extensions)
                __skb_ext_put(skb->extensions);
}

static inline void __skb_ext_copy(struct sk_buff *dst,
                                  const struct sk_buff *src)
{
        dst->active_extensions = src->active_extensions;

        if (src->active_extensions) {
                struct skb_ext *ext = src->extensions;

                refcount_inc(&ext->refcnt);
                dst->extensions = ext;
        }
}

static inline void skb_ext_copy(struct sk_buff *dst, const struct sk_buff *src)
{
        skb_ext_put(dst);
        __skb_ext_copy(dst, src);
}

static inline bool __skb_ext_exist(const struct skb_ext *ext, enum skb_ext_id i)
{
        return !!ext->offset[i];
}

static inline bool skb_ext_exist(const struct sk_buff *skb, enum skb_ext_id id)
{
        return skb->active_extensions & (1 << id);
}

static inline void skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
{
        if (skb_ext_exist(skb, id))
                __skb_ext_del(skb, id);
}

static inline void *skb_ext_find(const struct sk_buff *skb, enum skb_ext_id id)
{
        if (skb_ext_exist(skb, id)) {
                struct skb_ext *ext = skb->extensions;

                return (void *)ext + (ext->offset[id] << 3);
        }

        return NULL;
}

static inline void skb_ext_reset(struct sk_buff *skb)
{
        if (unlikely(skb->active_extensions)) {
                __skb_ext_put(skb->extensions);
                skb->active_extensions = 0;
        }
}

static inline bool skb_has_extensions(struct sk_buff *skb)
{
        return unlikely(skb->active_extensions);
}
#else
static inline void skb_ext_put(struct sk_buff *skb) {}
static inline void skb_ext_reset(struct sk_buff *skb) {}
static inline void skb_ext_del(struct sk_buff *skb, int unused) {}
static inline void __skb_ext_copy(struct sk_buff *d, const struct sk_buff *s) {}
static inline void skb_ext_copy(struct sk_buff *dst, const struct sk_buff *s) {}
static inline bool skb_has_extensions(struct sk_buff *skb) { return false; }
#endif /* CONFIG_SKB_EXTENSIONS */

static inline void nf_reset_ct(struct sk_buff *skb)
{
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
        nf_conntrack_put(skb_nfct(skb));
        skb->_nfct = 0;
#endif
}

static inline void nf_reset_trace(struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || IS_ENABLED(CONFIG_NF_TABLES)
        skb->nf_trace = 0;
#endif
}

static inline void ipvs_reset(struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_IP_VS)
        skb->ipvs_property = 0;
#endif
}

/* Note: This doesn't put any conntrack info in dst. */
static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src,
                             bool copy)
{
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
        dst->_nfct = src->_nfct;
        nf_conntrack_get(skb_nfct(src));
#endif
#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || IS_ENABLED(CONFIG_NF_TABLES)
        if (copy)
                dst->nf_trace = src->nf_trace;
#endif
}

static inline void nf_copy(struct sk_buff *dst, const struct sk_buff *src)
{
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
        nf_conntrack_put(skb_nfct(dst));
#endif
        dst->slow_gro = src->slow_gro;
        __nf_copy(dst, src, true);
}

#ifdef CONFIG_NETWORK_SECMARK
static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
{
        to->secmark = from->secmark;
}

static inline void skb_init_secmark(struct sk_buff *skb)
{
        skb->secmark = 0;
}
#else
static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
{ }

static inline void skb_init_secmark(struct sk_buff *skb)
{ }
#endif

static inline int secpath_exists(const struct sk_buff *skb)
{
#ifdef CONFIG_XFRM
        return skb_ext_exist(skb, SKB_EXT_SEC_PATH);
#else
        return 0;
#endif
}

static inline bool skb_irq_freeable(const struct sk_buff *skb)
{
        return !skb->destructor &&
                !secpath_exists(skb) &&
                !skb_nfct(skb) &&
                !skb->_skb_refdst &&
                !skb_has_frag_list(skb);
}

static inline void skb_set_queue_mapping(struct sk_buff *skb, u16 queue_mapping)
{
        skb->queue_mapping = queue_mapping;
}

static inline u16 skb_get_queue_mapping(const struct sk_buff *skb)
{
        return skb->queue_mapping;
}

static inline void skb_copy_queue_mapping(struct sk_buff *to, const struct sk_buff *from)
{
        to->queue_mapping = from->queue_mapping;
}

static inline void skb_record_rx_queue(struct sk_buff *skb, u16 rx_queue)
{
        skb->queue_mapping = rx_queue + 1;
}

static inline u16 skb_get_rx_queue(const struct sk_buff *skb)
{
        return skb->queue_mapping - 1;
}

static inline bool skb_rx_queue_recorded(const struct sk_buff *skb)
{
        return skb->queue_mapping != 0;
}

static inline void skb_set_dst_pending_confirm(struct sk_buff *skb, u32 val)
{
        skb->dst_pending_confirm = val;
}

static inline bool skb_get_dst_pending_confirm(const struct sk_buff *skb)
{
        return skb->dst_pending_confirm != 0;
}

static inline struct sec_path *skb_sec_path(const struct sk_buff *skb)
{
#ifdef CONFIG_XFRM
        return skb_ext_find(skb, SKB_EXT_SEC_PATH);
#else
        return NULL;
#endif
}

static inline bool skb_is_gso(const struct sk_buff *skb)
{
        return skb_shinfo(skb)->gso_size;
}

/* Note: Should be called only if skb_is_gso(skb) is true */
static inline bool skb_is_gso_v6(const struct sk_buff *skb)
{
        return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6;
}

/* Note: Should be called only if skb_is_gso(skb) is true */
static inline bool skb_is_gso_sctp(const struct sk_buff *skb)
{
        return skb_shinfo(skb)->gso_type & SKB_GSO_SCTP;
}

/* Note: Should be called only if skb_is_gso(skb) is true */
static inline bool skb_is_gso_tcp(const struct sk_buff *skb)
{
        return skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6);
}

static inline void skb_gso_reset(struct sk_buff *skb)
{
        skb_shinfo(skb)->gso_size = 0;
        skb_shinfo(skb)->gso_segs = 0;
        skb_shinfo(skb)->gso_type = 0;
}

static inline void skb_increase_gso_size(struct skb_shared_info *shinfo,
                                         u16 increment)
{
        if (WARN_ON_ONCE(shinfo->gso_size == GSO_BY_FRAGS))
                return;
        shinfo->gso_size += increment;
}

static inline void skb_decrease_gso_size(struct skb_shared_info *shinfo,
                                         u16 decrement)
{
        if (WARN_ON_ONCE(shinfo->gso_size == GSO_BY_FRAGS))
                return;
        shinfo->gso_size -= decrement;
}

void __skb_warn_lro_forwarding(const struct sk_buff *skb);

static inline bool skb_warn_if_lro(const struct sk_buff *skb)
{
        /* LRO sets gso_size but not gso_type, whereas if GSO is really
         * wanted then gso_type will be set. */
        const struct skb_shared_info *shinfo = skb_shinfo(skb);

        if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 &&
            unlikely(shinfo->gso_type == 0)) {
                __skb_warn_lro_forwarding(skb);
                return true;
        }
        return false;
}

static inline void skb_forward_csum(struct sk_buff *skb)
{
        /* Unfortunately we don't support this one.  Any brave souls? */
        if (skb->ip_summed == CHECKSUM_COMPLETE)
                skb->ip_summed = CHECKSUM_NONE;
}

/**
 * skb_checksum_none_assert - make sure skb ip_summed is CHECKSUM_NONE
 * @skb: skb to check
 *
 * fresh skbs have their ip_summed set to CHECKSUM_NONE.
 * Instead of forcing ip_summed to CHECKSUM_NONE, we can
 * use this helper, to document places where we make this assertion.
 */
static inline void skb_checksum_none_assert(const struct sk_buff *skb)
{
        DEBUG_NET_WARN_ON_ONCE(skb->ip_summed != CHECKSUM_NONE);
}

bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off);

int skb_checksum_setup(struct sk_buff *skb, bool recalculate);
struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
                                     unsigned int transport_len,
                                     __sum16(*skb_chkf)(struct sk_buff *skb));

/**
 * skb_head_is_locked - Determine if the skb->head is locked down
 * @skb: skb to check
 *
 * The head on skbs build around a head frag can be removed if they are
 * not cloned.  This function returns true if the skb head is locked down
 * due to either being allocated via kmalloc, or by being a clone with
 * multiple references to the head.
 */
static inline bool skb_head_is_locked(const struct sk_buff *skb)
{
        return !skb->head_frag || skb_cloned(skb);
}

/* Local Checksum Offload.
 * Compute outer checksum based on the assumption that the
 * inner checksum will be offloaded later.
 * See Documentation/networking/checksum-offloads.rst for
 * explanation of how this works.
 * Fill in outer checksum adjustment (e.g. with sum of outer
 * pseudo-header) before calling.
 * Also ensure that inner checksum is in linear data area.
 */
static inline __wsum lco_csum(struct sk_buff *skb)
{
        unsigned char *csum_start = skb_checksum_start(skb);
        unsigned char *l4_hdr = skb_transport_header(skb);
        __wsum partial;

        /* Start with complement of inner checksum adjustment */
        partial = ~csum_unfold(*(__force __sum16 *)(csum_start +
                                                    skb->csum_offset));

        /* Add in checksum of our headers (incl. outer checksum
         * adjustment filled in by caller) and return result.
         */
        return csum_partial(l4_hdr, csum_start - l4_hdr, partial);
}

static inline bool skb_is_redirected(const struct sk_buff *skb)
{
        return skb->redirected;
}

static inline void skb_set_redirected(struct sk_buff *skb, bool from_ingress)
{
        skb->redirected = 1;
#ifdef CONFIG_NET_REDIRECT
        skb->from_ingress = from_ingress;
        if (skb->from_ingress)
                skb_clear_tstamp(skb);
#endif
}

static inline void skb_reset_redirect(struct sk_buff *skb)
{
        skb->redirected = 0;
}

static inline void skb_set_redirected_noclear(struct sk_buff *skb,
                                              bool from_ingress)
{
        skb->redirected = 1;
#ifdef CONFIG_NET_REDIRECT
        skb->from_ingress = from_ingress;
#endif
}

static inline bool skb_csum_is_sctp(struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_IP_SCTP)
        return skb->csum_not_inet;
#else
        return 0;
#endif
}

static inline void skb_reset_csum_not_inet(struct sk_buff *skb)
{
        skb->ip_summed = CHECKSUM_NONE;
#if IS_ENABLED(CONFIG_IP_SCTP)
        skb->csum_not_inet = 0;
#endif
}

static inline void skb_set_kcov_handle(struct sk_buff *skb,
                                       const u64 kcov_handle)
{
#ifdef CONFIG_KCOV
        skb->kcov_handle = kcov_handle;
#endif
}

static inline u64 skb_get_kcov_handle(struct sk_buff *skb)
{
#ifdef CONFIG_KCOV
        return skb->kcov_handle;
#else
        return 0;
#endif
}

static inline void skb_mark_for_recycle(struct sk_buff *skb)
{
#ifdef CONFIG_PAGE_POOL
        skb->pp_recycle = 1;
#endif
}

ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter,
                             ssize_t maxsize, gfp_t gfp);

#endif        /* __KERNEL__ */
#endif        /* _LINUX_SKBUFF_H */


































































































































  182 




   21 






















































































































































   10 





   30 

















































































































   30 














1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* Internal procfs definitions
 *
 * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#include <linux/proc_fs.h>
#include <linux/proc_ns.h>
#include <linux/refcount.h>
#include <linux/spinlock.h>
#include <linux/atomic.h>
#include <linux/binfmts.h>
#include <linux/sched/coredump.h>
#include <linux/sched/task.h>
#include <linux/mm.h>

struct ctl_table_header;
struct mempolicy;

/*
 * This is not completely implemented yet. The idea is to
 * create an in-memory tree (like the actual /proc filesystem
 * tree) of these proc_dir_entries, so that we can dynamically
 * add new files to /proc.
 *
 * parent/subdir are used for the directory structure (every /proc file has a
 * parent, but "subdir" is empty for all non-directory entries).
 * subdir_node is used to build the rb tree "subdir" of the parent.
 */
struct proc_dir_entry {
        /*
         * number of callers into module in progress;
         * negative -> it's going away RSN
         */
        atomic_t in_use;
        refcount_t refcnt;
        struct list_head pde_openers;        /* who did ->open, but not ->release */
        /* protects ->pde_openers and all struct pde_opener instances */
        spinlock_t pde_unload_lock;
        struct completion *pde_unload_completion;
        const struct inode_operations *proc_iops;
        union {
                const struct proc_ops *proc_ops;
                const struct file_operations *proc_dir_ops;
        };
        const struct dentry_operations *proc_dops;
        union {
                const struct seq_operations *seq_ops;
                int (*single_show)(struct seq_file *, void *);
        };
        proc_write_t write;
        void *data;
        unsigned int state_size;
        unsigned int low_ino;
        nlink_t nlink;
        kuid_t uid;
        kgid_t gid;
        loff_t size;
        struct proc_dir_entry *parent;
        struct rb_root subdir;
        struct rb_node subdir_node;
        char *name;
        umode_t mode;
        u8 flags;
        u8 namelen;
        char inline_name[];
} __randomize_layout;

#define SIZEOF_PDE        (                                \
        sizeof(struct proc_dir_entry) < 128 ? 128 :        \
        sizeof(struct proc_dir_entry) < 192 ? 192 :        \
        sizeof(struct proc_dir_entry) < 256 ? 256 :        \
        sizeof(struct proc_dir_entry) < 512 ? 512 :        \
        0)
#define SIZEOF_PDE_INLINE_NAME (SIZEOF_PDE - sizeof(struct proc_dir_entry))

static inline bool pde_is_permanent(const struct proc_dir_entry *pde)
{
        return pde->flags & PROC_ENTRY_PERMANENT;
}

static inline void pde_make_permanent(struct proc_dir_entry *pde)
{
        pde->flags |= PROC_ENTRY_PERMANENT;
}

static inline bool pde_has_proc_read_iter(const struct proc_dir_entry *pde)
{
        return pde->flags & PROC_ENTRY_proc_read_iter;
}

static inline bool pde_has_proc_compat_ioctl(const struct proc_dir_entry *pde)
{
#ifdef CONFIG_COMPAT
        return pde->flags & PROC_ENTRY_proc_compat_ioctl;
#else
        return false;
#endif
}

extern struct kmem_cache *proc_dir_entry_cache;
void pde_free(struct proc_dir_entry *pde);

union proc_op {
        int (*proc_get_link)(struct dentry *, struct path *);
        int (*proc_show)(struct seq_file *m,
                struct pid_namespace *ns, struct pid *pid,
                struct task_struct *task);
        int lsmid;
};

struct proc_inode {
        struct pid *pid;
        unsigned int fd;
        union proc_op op;
        struct proc_dir_entry *pde;
        struct ctl_table_header *sysctl;
        const struct ctl_table *sysctl_entry;
        struct hlist_node sibling_inodes;
        const struct proc_ns_operations *ns_ops;
        struct inode vfs_inode;
} __randomize_layout;

/*
 * General functions
 */
static inline struct proc_inode *PROC_I(const struct inode *inode)
{
        return container_of(inode, struct proc_inode, vfs_inode);
}

static inline struct proc_dir_entry *PDE(const struct inode *inode)
{
        return PROC_I(inode)->pde;
}

static inline struct pid *proc_pid(const struct inode *inode)
{
        return PROC_I(inode)->pid;
}

static inline struct task_struct *get_proc_task(const struct inode *inode)
{
        return get_pid_task(proc_pid(inode), PIDTYPE_PID);
}

void task_dump_owner(struct task_struct *task, umode_t mode,
                     kuid_t *ruid, kgid_t *rgid);

unsigned name_to_int(const struct qstr *qstr);
/*
 * Offset of the first process in the /proc root directory..
 */
#define FIRST_PROCESS_ENTRY 256

/* Worst case buffer size needed for holding an integer. */
#define PROC_NUMBUF 13

#ifdef CONFIG_PAGE_MAPCOUNT
/**
 * folio_precise_page_mapcount() - Number of mappings of this folio page.
 * @folio: The folio.
 * @page: The page.
 *
 * The number of present user page table entries that reference this page
 * as tracked via the RMAP: either referenced directly (PTE) or as part of
 * a larger area that covers this page (e.g., PMD).
 *
 * Use this function only for the calculation of existing statistics
 * (USS, PSS, mapcount_max) and for debugging purposes (/proc/kpagecount).
 *
 * Do not add new users.
 *
 * Returns: The number of mappings of this folio page. 0 for
 * folios that are not mapped to user space or are not tracked via the RMAP
 * (e.g., shared zeropage).
 */
static inline int folio_precise_page_mapcount(struct folio *folio,
                struct page *page)
{
        int mapcount = atomic_read(&page->_mapcount) + 1;

        if (page_mapcount_is_type(mapcount))
                mapcount = 0;
        if (folio_test_large(folio))
                mapcount += folio_entire_mapcount(folio);

        return mapcount;
}
#else /* !CONFIG_PAGE_MAPCOUNT */
static inline int folio_precise_page_mapcount(struct folio *folio,
                struct page *page)
{
        BUILD_BUG();
}
#endif /* CONFIG_PAGE_MAPCOUNT */

/**
 * folio_average_page_mapcount() - Average number of mappings per page in this
 *                                   folio
 * @folio: The folio.
 *
 * The average number of user page table entries that reference each page in
 * this folio as tracked via the RMAP: either referenced directly (PTE) or
 * as part of a larger area that covers this page (e.g., PMD).
 *
 * The average is calculated by rounding to the nearest integer; however,
 * to avoid duplicated code in current callers, the average is at least
 * 1 if any page of the folio is mapped.
 *
 * Returns: The average number of mappings per page in this folio.
 */
static inline int folio_average_page_mapcount(struct folio *folio)
{
        int mapcount, entire_mapcount, avg;

        if (!folio_test_large(folio))
                return atomic_read(&folio->_mapcount) + 1;

        mapcount = folio_large_mapcount(folio);
        if (unlikely(mapcount <= 0))
                return 0;
        entire_mapcount = folio_entire_mapcount(folio);
        if (mapcount <= entire_mapcount)
                return entire_mapcount;
        mapcount -= entire_mapcount;

        /* Round to closest integer ... */
        avg = ((unsigned int)mapcount + folio_large_nr_pages(folio) / 2) >> folio_large_order(folio);
        /* ... but return at least 1. */
        return max_t(int, avg + entire_mapcount, 1);
}
/*
 * array.c
 */
extern const struct file_operations proc_tid_children_operations;

extern void proc_task_name(struct seq_file *m, struct task_struct *p,
                           bool escape);
extern int proc_tid_stat(struct seq_file *, struct pid_namespace *,
                         struct pid *, struct task_struct *);
extern int proc_tgid_stat(struct seq_file *, struct pid_namespace *,
                          struct pid *, struct task_struct *);
extern int proc_pid_status(struct seq_file *, struct pid_namespace *,
                           struct pid *, struct task_struct *);
extern int proc_pid_statm(struct seq_file *, struct pid_namespace *,
                          struct pid *, struct task_struct *);

/*
 * base.c
 */
extern const struct dentry_operations pid_dentry_operations;
extern int pid_getattr(struct mnt_idmap *, const struct path *,
                       struct kstat *, u32, unsigned int);
extern int proc_setattr(struct mnt_idmap *, struct dentry *,
                        struct iattr *);
extern void proc_pid_evict_inode(struct proc_inode *);
extern struct inode *proc_pid_make_inode(struct super_block *, struct task_struct *, umode_t);
extern void pid_update_inode(struct task_struct *, struct inode *);
extern int pid_delete_dentry(const struct dentry *);
extern int proc_pid_readdir(struct file *, struct dir_context *);
struct dentry *proc_pid_lookup(struct dentry *, unsigned int);
extern loff_t mem_lseek(struct file *, loff_t, int);

/* Lookups */
typedef struct dentry *instantiate_t(struct dentry *,
                                     struct task_struct *, const void *);
bool proc_fill_cache(struct file *, struct dir_context *, const char *, unsigned int,
                           instantiate_t, struct task_struct *, const void *);

/*
 * generic.c
 */
struct proc_dir_entry *proc_create_reg(const char *name, umode_t mode,
                struct proc_dir_entry **parent, void *data);
struct proc_dir_entry *proc_register(struct proc_dir_entry *dir,
                struct proc_dir_entry *dp);
extern struct dentry *proc_lookup(struct inode *, struct dentry *, unsigned int);
struct dentry *proc_lookup_de(struct inode *, struct dentry *, struct proc_dir_entry *);
extern int proc_readdir(struct file *, struct dir_context *);
int proc_readdir_de(struct file *, struct dir_context *, struct proc_dir_entry *);

static inline void pde_get(struct proc_dir_entry *pde)
{
        refcount_inc(&pde->refcnt);
}
extern void pde_put(struct proc_dir_entry *);

static inline bool is_empty_pde(const struct proc_dir_entry *pde)
{
        return S_ISDIR(pde->mode) && !pde->proc_iops;
}
extern ssize_t proc_simple_write(struct file *, const char __user *, size_t, loff_t *);

/*
 * inode.c
 */
struct pde_opener {
        struct list_head lh;
        struct file *file;
        bool closing;
        struct completion *c;
} __randomize_layout;
extern const struct inode_operations proc_link_inode_operations;
extern const struct inode_operations proc_pid_link_inode_operations;
extern const struct super_operations proc_sops;

void proc_init_kmemcache(void);
void proc_invalidate_siblings_dcache(struct hlist_head *inodes, spinlock_t *lock);
void set_proc_pid_nlink(void);
extern struct inode *proc_get_inode(struct super_block *, struct proc_dir_entry *);
extern void proc_entry_rundown(struct proc_dir_entry *);

/*
 * proc_namespaces.c
 */
extern const struct inode_operations proc_ns_dir_inode_operations;
extern const struct file_operations proc_ns_dir_operations;

/*
 * proc_net.c
 */
extern const struct file_operations proc_net_operations;
extern const struct inode_operations proc_net_inode_operations;

#ifdef CONFIG_NET
extern int proc_net_init(void);
#else
static inline int proc_net_init(void) { return 0; }
#endif

/*
 * proc_self.c
 */
extern int proc_setup_self(struct super_block *);

/*
 * proc_thread_self.c
 */
extern int proc_setup_thread_self(struct super_block *);
extern void proc_thread_self_init(void);

/*
 * proc_sysctl.c
 */
#ifdef CONFIG_PROC_SYSCTL
extern int proc_sys_init(void);
extern void proc_sys_evict_inode(struct inode *inode,
                                 struct ctl_table_header *head);
#else
static inline void proc_sys_init(void) { }
static inline void proc_sys_evict_inode(struct  inode *inode,
                                        struct ctl_table_header *head) { }
#endif

/*
 * proc_tty.c
 */
#ifdef CONFIG_TTY
extern void proc_tty_init(void);
#else
static inline void proc_tty_init(void) {}
#endif

/*
 * root.c
 */
extern struct proc_dir_entry proc_root;

extern void proc_self_init(void);

/*
 * task_[no]mmu.c
 */
struct mem_size_stats;
struct proc_maps_private {
        struct inode *inode;
        struct task_struct *task;
        struct mm_struct *mm;
        struct vma_iterator iter;
#ifdef CONFIG_NUMA
        struct mempolicy *task_mempolicy;
#endif
} __randomize_layout;

struct mm_struct *proc_mem_open(struct inode *inode, unsigned int mode);

extern const struct file_operations proc_pid_maps_operations;
extern const struct file_operations proc_pid_numa_maps_operations;
extern const struct file_operations proc_pid_smaps_operations;
extern const struct file_operations proc_pid_smaps_rollup_operations;
extern const struct file_operations proc_clear_refs_operations;
extern const struct file_operations proc_pagemap_operations;

extern unsigned long task_vsize(struct mm_struct *);
extern unsigned long task_statm(struct mm_struct *,
                                unsigned long *, unsigned long *,
                                unsigned long *, unsigned long *);
extern void task_mem(struct seq_file *, struct mm_struct *);

extern const struct dentry_operations proc_net_dentry_ops;
static inline void pde_force_lookup(struct proc_dir_entry *pde)
{
        /* /proc/net/ entries can be changed under us by setns(CLONE_NEWNET) */
        pde->proc_dops = &proc_net_dentry_ops;
}

/*
 * Add a new procfs dentry that can't serve as a mountpoint. That should
 * encompass anything that is ephemeral and can just disappear while the
 * process is still around.
 */
static inline struct dentry *proc_splice_unmountable(struct inode *inode,
                struct dentry *dentry, const struct dentry_operations *d_ops)
{
        d_set_d_op(dentry, d_ops);
        dont_mount(dentry);
        return d_splice_alias(inode, dentry);
}







































   29 
   29 



























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   29 










   29 











































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2000-2002 Joakim Axelsson <gozem@linux.nu>
 *                         Patrick Schaaf <bof@bof.de>
 * Copyright (C) 2003-2013 Jozsef Kadlecsik <kadlec@netfilter.org>
 */

/* Kernel module for IP set management */

#include <linux/init.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/ip.h>
#include <linux/skbuff.h>
#include <linux/spinlock.h>
#include <linux/rculist.h>
#include <net/netlink.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>

#include <linux/netfilter.h>
#include <linux/netfilter/x_tables.h>
#include <linux/netfilter/nfnetlink.h>
#include <linux/netfilter/ipset/ip_set.h>

static LIST_HEAD(ip_set_type_list);                /* all registered set types */
static DEFINE_MUTEX(ip_set_type_mutex);                /* protects ip_set_type_list */
static DEFINE_RWLOCK(ip_set_ref_lock);                /* protects the set refs */

struct ip_set_net {
        struct ip_set * __rcu *ip_set_list;        /* all individual sets */
        ip_set_id_t        ip_set_max;        /* max number of sets */
        bool                is_deleted;        /* deleted by ip_set_net_exit */
        bool                is_destroyed;        /* all sets are destroyed */
};

static unsigned int ip_set_net_id __read_mostly;

static struct ip_set_net *ip_set_pernet(struct net *net)
{
        return net_generic(net, ip_set_net_id);
}

#define IP_SET_INC        64
#define STRNCMP(a, b)        (strncmp(a, b, IPSET_MAXNAMELEN) == 0)

static unsigned int max_sets;

module_param(max_sets, int, 0600);
MODULE_PARM_DESC(max_sets, "maximal number of sets");
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Jozsef Kadlecsik <kadlec@netfilter.org>");
MODULE_DESCRIPTION("core IP set support");
MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_IPSET);

/* When the nfnl mutex or ip_set_ref_lock is held: */
#define ip_set_dereference(inst)        \
        rcu_dereference_protected((inst)->ip_set_list,        \
                lockdep_nfnl_is_held(NFNL_SUBSYS_IPSET) || \
                lockdep_is_held(&ip_set_ref_lock) || \
                (inst)->is_deleted)
#define ip_set(inst, id)                \
        ip_set_dereference(inst)[id]
#define ip_set_ref_netlink(inst,id)        \
        rcu_dereference_raw((inst)->ip_set_list)[id]
#define ip_set_dereference_nfnl(p)        \
        rcu_dereference_check(p, lockdep_nfnl_is_held(NFNL_SUBSYS_IPSET))

/* The set types are implemented in modules and registered set types
 * can be found in ip_set_type_list. Adding/deleting types is
 * serialized by ip_set_type_mutex.
 */

static void
ip_set_type_lock(void)
{
        mutex_lock(&ip_set_type_mutex);
}

static void
ip_set_type_unlock(void)
{
        mutex_unlock(&ip_set_type_mutex);
}

/* Register and deregister settype */

static struct ip_set_type *
find_set_type(const char *name, u8 family, u8 revision)
{
        struct ip_set_type *type;

        list_for_each_entry_rcu(type, &ip_set_type_list, list,
                                lockdep_is_held(&ip_set_type_mutex))
                if (STRNCMP(type->name, name) &&
                    (type->family == family ||
                     type->family == NFPROTO_UNSPEC) &&
                    revision >= type->revision_min &&
                    revision <= type->revision_max)
                        return type;
        return NULL;
}

/* Unlock, try to load a set type module and lock again */
static bool
load_settype(const char *name)
{
        if (!try_module_get(THIS_MODULE))
                return false;

        nfnl_unlock(NFNL_SUBSYS_IPSET);
        pr_debug("try to load ip_set_%s\n", name);
        if (request_module("ip_set_%s", name) < 0) {
                pr_warn("Can't find ip_set type %s\n", name);
                nfnl_lock(NFNL_SUBSYS_IPSET);
                module_put(THIS_MODULE);
                return false;
        }
        nfnl_lock(NFNL_SUBSYS_IPSET);
        module_put(THIS_MODULE);
        return true;
}

/* Find a set type and reference it */
#define find_set_type_get(name, family, revision, found)        \
        __find_set_type_get(name, family, revision, found, false)

static int
__find_set_type_get(const char *name, u8 family, u8 revision,
                    struct ip_set_type **found, bool retry)
{
        struct ip_set_type *type;
        int err;

        if (retry && !load_settype(name))
                return -IPSET_ERR_FIND_TYPE;

        rcu_read_lock();
        *found = find_set_type(name, family, revision);
        if (*found) {
                err = !try_module_get((*found)->me) ? -EFAULT : 0;
                goto unlock;
        }
        /* Make sure the type is already loaded
         * but we don't support the revision
         */
        list_for_each_entry_rcu(type, &ip_set_type_list, list)
                if (STRNCMP(type->name, name)) {
                        err = -IPSET_ERR_FIND_TYPE;
                        goto unlock;
                }
        rcu_read_unlock();

        return retry ? -IPSET_ERR_FIND_TYPE :
                __find_set_type_get(name, family, revision, found, true);

unlock:
        rcu_read_unlock();
        return err;
}

/* Find a given set type by name and family.
 * If we succeeded, the supported minimal and maximum revisions are
 * filled out.
 */
#define find_set_type_minmax(name, family, min, max) \
        __find_set_type_minmax(name, family, min, max, false)

static int
__find_set_type_minmax(const char *name, u8 family, u8 *min, u8 *max,
                       bool retry)
{
        struct ip_set_type *type;
        bool found = false;

        if (retry && !load_settype(name))
                return -IPSET_ERR_FIND_TYPE;

        *min = 255; *max = 0;
        rcu_read_lock();
        list_for_each_entry_rcu(type, &ip_set_type_list, list)
                if (STRNCMP(type->name, name) &&
                    (type->family == family ||
                     type->family == NFPROTO_UNSPEC)) {
                        found = true;
                        if (type->revision_min < *min)
                                *min = type->revision_min;
                        if (type->revision_max > *max)
                                *max = type->revision_max;
                }
        rcu_read_unlock();
        if (found)
                return 0;

        return retry ? -IPSET_ERR_FIND_TYPE :
                __find_set_type_minmax(name, family, min, max, true);
}

#define family_name(f)        ((f) == NFPROTO_IPV4 ? "inet" : \
                         (f) == NFPROTO_IPV6 ? "inet6" : "any")

/* Register a set type structure. The type is identified by
 * the unique triple of name, family and revision.
 */
int
ip_set_type_register(struct ip_set_type *type)
{
        int ret = 0;

        if (type->protocol != IPSET_PROTOCOL) {
                pr_warn("ip_set type %s, family %s, revision %u:%u uses wrong protocol version %u (want %u)\n",
                        type->name, family_name(type->family),
                        type->revision_min, type->revision_max,
                        type->protocol, IPSET_PROTOCOL);
                return -EINVAL;
        }

        ip_set_type_lock();
        if (find_set_type(type->name, type->family, type->revision_min)) {
                /* Duplicate! */
                pr_warn("ip_set type %s, family %s with revision min %u already registered!\n",
                        type->name, family_name(type->family),
                        type->revision_min);
                ip_set_type_unlock();
                return -EINVAL;
        }
        list_add_rcu(&type->list, &ip_set_type_list);
        pr_debug("type %s, family %s, revision %u:%u registered.\n",
                 type->name, family_name(type->family),
                 type->revision_min, type->revision_max);
        ip_set_type_unlock();

        return ret;
}
EXPORT_SYMBOL_GPL(ip_set_type_register);

/* Unregister a set type. There's a small race with ip_set_create */
void
ip_set_type_unregister(struct ip_set_type *type)
{
        ip_set_type_lock();
        if (!find_set_type(type->name, type->family, type->revision_min)) {
                pr_warn("ip_set type %s, family %s with revision min %u not registered\n",
                        type->name, family_name(type->family),
                        type->revision_min);
                ip_set_type_unlock();
                return;
        }
        list_del_rcu(&type->list);
        pr_debug("type %s, family %s with revision min %u unregistered.\n",
                 type->name, family_name(type->family), type->revision_min);
        ip_set_type_unlock();

        synchronize_rcu();
}
EXPORT_SYMBOL_GPL(ip_set_type_unregister);

/* Utility functions */
void *
ip_set_alloc(size_t size)
{
        return kvzalloc(size, GFP_KERNEL_ACCOUNT);
}
EXPORT_SYMBOL_GPL(ip_set_alloc);

void
ip_set_free(void *members)
{
        pr_debug("%p: free with %s\n", members,
                 is_vmalloc_addr(members) ? "vfree" : "kfree");
        kvfree(members);
}
EXPORT_SYMBOL_GPL(ip_set_free);

static bool
flag_nested(const struct nlattr *nla)
{
        return nla->nla_type & NLA_F_NESTED;
}

static const struct nla_policy ipaddr_policy[IPSET_ATTR_IPADDR_MAX + 1] = {
        [IPSET_ATTR_IPADDR_IPV4]        = { .type = NLA_U32 },
        [IPSET_ATTR_IPADDR_IPV6]        = NLA_POLICY_EXACT_LEN(sizeof(struct in6_addr)),
};

int
ip_set_get_ipaddr4(struct nlattr *nla,  __be32 *ipaddr)
{
        struct nlattr *tb[IPSET_ATTR_IPADDR_MAX + 1];

        if (unlikely(!flag_nested(nla)))
                return -IPSET_ERR_PROTOCOL;
        if (nla_parse_nested(tb, IPSET_ATTR_IPADDR_MAX, nla,
                             ipaddr_policy, NULL))
                return -IPSET_ERR_PROTOCOL;
        if (unlikely(!ip_set_attr_netorder(tb, IPSET_ATTR_IPADDR_IPV4)))
                return -IPSET_ERR_PROTOCOL;

        *ipaddr = nla_get_be32(tb[IPSET_ATTR_IPADDR_IPV4]);
        return 0;
}
EXPORT_SYMBOL_GPL(ip_set_get_ipaddr4);

int
ip_set_get_ipaddr6(struct nlattr *nla, union nf_inet_addr *ipaddr)
{
        struct nlattr *tb[IPSET_ATTR_IPADDR_MAX + 1];

        if (unlikely(!flag_nested(nla)))
                return -IPSET_ERR_PROTOCOL;

        if (nla_parse_nested(tb, IPSET_ATTR_IPADDR_MAX, nla,
                             ipaddr_policy, NULL))
                return -IPSET_ERR_PROTOCOL;
        if (unlikely(!ip_set_attr_netorder(tb, IPSET_ATTR_IPADDR_IPV6)))
                return -IPSET_ERR_PROTOCOL;

        memcpy(ipaddr, nla_data(tb[IPSET_ATTR_IPADDR_IPV6]),
               sizeof(struct in6_addr));
        return 0;
}
EXPORT_SYMBOL_GPL(ip_set_get_ipaddr6);

static u32
ip_set_timeout_get(const unsigned long *timeout)
{
        u32 t;

        if (*timeout == IPSET_ELEM_PERMANENT)
                return 0;

        t = jiffies_to_msecs(*timeout - jiffies) / MSEC_PER_SEC;
        /* Zero value in userspace means no timeout */
        return t == 0 ? 1 : t;
}

static char *
ip_set_comment_uget(struct nlattr *tb)
{
        return nla_data(tb);
}

/* Called from uadd only, protected by the set spinlock.
 * The kadt functions don't use the comment extensions in any way.
 */
void
ip_set_init_comment(struct ip_set *set, struct ip_set_comment *comment,
                    const struct ip_set_ext *ext)
{
        struct ip_set_comment_rcu *c = rcu_dereference_protected(comment->c, 1);
        size_t len = ext->comment ? strlen(ext->comment) : 0;

        if (unlikely(c)) {
                set->ext_size -= sizeof(*c) + strlen(c->str) + 1;
                kfree_rcu(c, rcu);
                rcu_assign_pointer(comment->c, NULL);
        }
        if (!len)
                return;
        if (unlikely(len > IPSET_MAX_COMMENT_SIZE))
                len = IPSET_MAX_COMMENT_SIZE;
        c = kmalloc(sizeof(*c) + len + 1, GFP_ATOMIC);
        if (unlikely(!c))
                return;
        strscpy(c->str, ext->comment, len + 1);
        set->ext_size += sizeof(*c) + strlen(c->str) + 1;
        rcu_assign_pointer(comment->c, c);
}
EXPORT_SYMBOL_GPL(ip_set_init_comment);

/* Used only when dumping a set, protected by rcu_read_lock() */
static int
ip_set_put_comment(struct sk_buff *skb, const struct ip_set_comment *comment)
{
        struct ip_set_comment_rcu *c = rcu_dereference(comment->c);

        if (!c)
                return 0;
        return nla_put_string(skb, IPSET_ATTR_COMMENT, c->str);
}

/* Called from uadd/udel, flush or the garbage collectors protected
 * by the set spinlock.
 * Called when the set is destroyed and when there can't be any user
 * of the set data anymore.
 */
static void
ip_set_comment_free(struct ip_set *set, void *ptr)
{
        struct ip_set_comment *comment = ptr;
        struct ip_set_comment_rcu *c;

        c = rcu_dereference_protected(comment->c, 1);
        if (unlikely(!c))
                return;
        set->ext_size -= sizeof(*c) + strlen(c->str) + 1;
        kfree_rcu(c, rcu);
        rcu_assign_pointer(comment->c, NULL);
}

typedef void (*destroyer)(struct ip_set *, void *);
/* ipset data extension types, in size order */

const struct ip_set_ext_type ip_set_extensions[] = {
        [IPSET_EXT_ID_COUNTER] = {
                .type        = IPSET_EXT_COUNTER,
                .flag        = IPSET_FLAG_WITH_COUNTERS,
                .len        = sizeof(struct ip_set_counter),
                .align        = __alignof__(struct ip_set_counter),
        },
        [IPSET_EXT_ID_TIMEOUT] = {
                .type        = IPSET_EXT_TIMEOUT,
                .len        = sizeof(unsigned long),
                .align        = __alignof__(unsigned long),
        },
        [IPSET_EXT_ID_SKBINFO] = {
                .type        = IPSET_EXT_SKBINFO,
                .flag        = IPSET_FLAG_WITH_SKBINFO,
                .len        = sizeof(struct ip_set_skbinfo),
                .align        = __alignof__(struct ip_set_skbinfo),
        },
        [IPSET_EXT_ID_COMMENT] = {
                .type         = IPSET_EXT_COMMENT | IPSET_EXT_DESTROY,
                .flag         = IPSET_FLAG_WITH_COMMENT,
                .len         = sizeof(struct ip_set_comment),
                .align         = __alignof__(struct ip_set_comment),
                .destroy = ip_set_comment_free,
        },
};
EXPORT_SYMBOL_GPL(ip_set_extensions);

static bool
add_extension(enum ip_set_ext_id id, u32 flags, struct nlattr *tb[])
{
        return ip_set_extensions[id].flag ?
                (flags & ip_set_extensions[id].flag) :
                !!tb[IPSET_ATTR_TIMEOUT];
}

size_t
ip_set_elem_len(struct ip_set *set, struct nlattr *tb[], size_t len,
                size_t align)
{
        enum ip_set_ext_id id;
        u32 cadt_flags = 0;

        if (tb[IPSET_ATTR_CADT_FLAGS])
                cadt_flags = ip_set_get_h32(tb[IPSET_ATTR_CADT_FLAGS]);
        if (cadt_flags & IPSET_FLAG_WITH_FORCEADD)
                set->flags |= IPSET_CREATE_FLAG_FORCEADD;
        if (!align)
                align = 1;
        for (id = 0; id < IPSET_EXT_ID_MAX; id++) {
                if (!add_extension(id, cadt_flags, tb))
                        continue;
                if (align < ip_set_extensions[id].align)
                        align = ip_set_extensions[id].align;
                len = ALIGN(len, ip_set_extensions[id].align);
                set->offset[id] = len;
                set->extensions |= ip_set_extensions[id].type;
                len += ip_set_extensions[id].len;
        }
        return ALIGN(len, align);
}
EXPORT_SYMBOL_GPL(ip_set_elem_len);

int
ip_set_get_extensions(struct ip_set *set, struct nlattr *tb[],
                      struct ip_set_ext *ext)
{
        u64 fullmark;

        if (unlikely(!ip_set_optattr_netorder(tb, IPSET_ATTR_TIMEOUT) ||
                     !ip_set_optattr_netorder(tb, IPSET_ATTR_PACKETS) ||
                     !ip_set_optattr_netorder(tb, IPSET_ATTR_BYTES) ||
                     !ip_set_optattr_netorder(tb, IPSET_ATTR_SKBMARK) ||
                     !ip_set_optattr_netorder(tb, IPSET_ATTR_SKBPRIO) ||
                     !ip_set_optattr_netorder(tb, IPSET_ATTR_SKBQUEUE)))
                return -IPSET_ERR_PROTOCOL;

        if (tb[IPSET_ATTR_TIMEOUT]) {
                if (!SET_WITH_TIMEOUT(set))
                        return -IPSET_ERR_TIMEOUT;
                ext->timeout = ip_set_timeout_uget(tb[IPSET_ATTR_TIMEOUT]);
        }
        if (tb[IPSET_ATTR_BYTES] || tb[IPSET_ATTR_PACKETS]) {
                if (!SET_WITH_COUNTER(set))
                        return -IPSET_ERR_COUNTER;
                if (tb[IPSET_ATTR_BYTES])
                        ext->bytes = be64_to_cpu(nla_get_be64(
                                                 tb[IPSET_ATTR_BYTES]));
                if (tb[IPSET_ATTR_PACKETS])
                        ext->packets = be64_to_cpu(nla_get_be64(
                                                   tb[IPSET_ATTR_PACKETS]));
        }
        if (tb[IPSET_ATTR_COMMENT]) {
                if (!SET_WITH_COMMENT(set))
                        return -IPSET_ERR_COMMENT;
                ext->comment = ip_set_comment_uget(tb[IPSET_ATTR_COMMENT]);
        }
        if (tb[IPSET_ATTR_SKBMARK]) {
                if (!SET_WITH_SKBINFO(set))
                        return -IPSET_ERR_SKBINFO;
                fullmark = be64_to_cpu(nla_get_be64(tb[IPSET_ATTR_SKBMARK]));
                ext->skbinfo.skbmark = fullmark >> 32;
                ext->skbinfo.skbmarkmask = fullmark & 0xffffffff;
        }
        if (tb[IPSET_ATTR_SKBPRIO]) {
                if (!SET_WITH_SKBINFO(set))
                        return -IPSET_ERR_SKBINFO;
                ext->skbinfo.skbprio =
                        be32_to_cpu(nla_get_be32(tb[IPSET_ATTR_SKBPRIO]));
        }
        if (tb[IPSET_ATTR_SKBQUEUE]) {
                if (!SET_WITH_SKBINFO(set))
                        return -IPSET_ERR_SKBINFO;
                ext->skbinfo.skbqueue =
                        be16_to_cpu(nla_get_be16(tb[IPSET_ATTR_SKBQUEUE]));
        }
        return 0;
}
EXPORT_SYMBOL_GPL(ip_set_get_extensions);

static u64
ip_set_get_bytes(const struct ip_set_counter *counter)
{
        return (u64)atomic64_read(&(counter)->bytes);
}

static u64
ip_set_get_packets(const struct ip_set_counter *counter)
{
        return (u64)atomic64_read(&(counter)->packets);
}

static bool
ip_set_put_counter(struct sk_buff *skb, const struct ip_set_counter *counter)
{
        return nla_put_net64(skb, IPSET_ATTR_BYTES,
                             cpu_to_be64(ip_set_get_bytes(counter)),
                             IPSET_ATTR_PAD) ||
               nla_put_net64(skb, IPSET_ATTR_PACKETS,
                             cpu_to_be64(ip_set_get_packets(counter)),
                             IPSET_ATTR_PAD);
}

static bool
ip_set_put_skbinfo(struct sk_buff *skb, const struct ip_set_skbinfo *skbinfo)
{
        /* Send nonzero parameters only */
        return ((skbinfo->skbmark || skbinfo->skbmarkmask) &&
                nla_put_net64(skb, IPSET_ATTR_SKBMARK,
                              cpu_to_be64((u64)skbinfo->skbmark << 32 |
                                          skbinfo->skbmarkmask),
                              IPSET_ATTR_PAD)) ||
               (skbinfo->skbprio &&
                nla_put_net32(skb, IPSET_ATTR_SKBPRIO,
                              cpu_to_be32(skbinfo->skbprio))) ||
               (skbinfo->skbqueue &&
                nla_put_net16(skb, IPSET_ATTR_SKBQUEUE,
                              cpu_to_be16(skbinfo->skbqueue)));
}

int
ip_set_put_extensions(struct sk_buff *skb, const struct ip_set *set,
                      const void *e, bool active)
{
        if (SET_WITH_TIMEOUT(set)) {
                unsigned long *timeout = ext_timeout(e, set);

                if (nla_put_net32(skb, IPSET_ATTR_TIMEOUT,
                        htonl(active ? ip_set_timeout_get(timeout)
                                : *timeout)))
                        return -EMSGSIZE;
        }
        if (SET_WITH_COUNTER(set) &&
            ip_set_put_counter(skb, ext_counter(e, set)))
                return -EMSGSIZE;
        if (SET_WITH_COMMENT(set) &&
            ip_set_put_comment(skb, ext_comment(e, set)))
                return -EMSGSIZE;
        if (SET_WITH_SKBINFO(set) &&
            ip_set_put_skbinfo(skb, ext_skbinfo(e, set)))
                return -EMSGSIZE;
        return 0;
}
EXPORT_SYMBOL_GPL(ip_set_put_extensions);

static bool
ip_set_match_counter(u64 counter, u64 match, u8 op)
{
        switch (op) {
        case IPSET_COUNTER_NONE:
                return true;
        case IPSET_COUNTER_EQ:
                return counter == match;
        case IPSET_COUNTER_NE:
                return counter != match;
        case IPSET_COUNTER_LT:
                return counter < match;
        case IPSET_COUNTER_GT:
                return counter > match;
        }
        return false;
}

static void
ip_set_add_bytes(u64 bytes, struct ip_set_counter *counter)
{
        atomic64_add((long long)bytes, &(counter)->bytes);
}

static void
ip_set_add_packets(u64 packets, struct ip_set_counter *counter)
{
        atomic64_add((long long)packets, &(counter)->packets);
}

static void
ip_set_update_counter(struct ip_set_counter *counter,
                      const struct ip_set_ext *ext, u32 flags)
{
        if (ext->packets != ULLONG_MAX &&
            !(flags & IPSET_FLAG_SKIP_COUNTER_UPDATE)) {
                ip_set_add_bytes(ext->bytes, counter);
                ip_set_add_packets(ext->packets, counter);
        }
}

static void
ip_set_get_skbinfo(struct ip_set_skbinfo *skbinfo,
                   const struct ip_set_ext *ext,
                   struct ip_set_ext *mext, u32 flags)
{
        mext->skbinfo = *skbinfo;
}

bool
ip_set_match_extensions(struct ip_set *set, const struct ip_set_ext *ext,
                        struct ip_set_ext *mext, u32 flags, void *data)
{
        if (SET_WITH_TIMEOUT(set) &&
            ip_set_timeout_expired(ext_timeout(data, set)))
                return false;
        if (SET_WITH_COUNTER(set)) {
                struct ip_set_counter *counter = ext_counter(data, set);

                ip_set_update_counter(counter, ext, flags);

                if (flags & IPSET_FLAG_MATCH_COUNTERS &&
                    !(ip_set_match_counter(ip_set_get_packets(counter),
                                mext->packets, mext->packets_op) &&
                      ip_set_match_counter(ip_set_get_bytes(counter),
                                mext->bytes, mext->bytes_op)))
                        return false;
        }
        if (SET_WITH_SKBINFO(set))
                ip_set_get_skbinfo(ext_skbinfo(data, set),
                                   ext, mext, flags);
        return true;
}
EXPORT_SYMBOL_GPL(ip_set_match_extensions);

/* Creating/destroying/renaming/swapping affect the existence and
 * the properties of a set. All of these can be executed from userspace
 * only and serialized by the nfnl mutex indirectly from nfnetlink.
 *
 * Sets are identified by their index in ip_set_list and the index
 * is used by the external references (set/SET netfilter modules).
 *
 * The set behind an index may change by swapping only, from userspace.
 */

static void
__ip_set_get(struct ip_set *set)
{
        write_lock_bh(&ip_set_ref_lock);
        set->ref++;
        write_unlock_bh(&ip_set_ref_lock);
}

static void
__ip_set_put(struct ip_set *set)
{
        write_lock_bh(&ip_set_ref_lock);
        BUG_ON(set->ref == 0);
        set->ref--;
        write_unlock_bh(&ip_set_ref_lock);
}

/* set->ref can be swapped out by ip_set_swap, netlink events (like dump) need
 * a separate reference counter
 */
static void
__ip_set_get_netlink(struct ip_set *set)
{
        write_lock_bh(&ip_set_ref_lock);
        set->ref_netlink++;
        write_unlock_bh(&ip_set_ref_lock);
}

static void
__ip_set_put_netlink(struct ip_set *set)
{
        write_lock_bh(&ip_set_ref_lock);
        BUG_ON(set->ref_netlink == 0);
        set->ref_netlink--;
        write_unlock_bh(&ip_set_ref_lock);
}

/* Add, del and test set entries from kernel.
 *
 * The set behind the index must exist and must be referenced
 * so it can't be destroyed (or changed) under our foot.
 */

static struct ip_set *
ip_set_rcu_get(struct net *net, ip_set_id_t index)
{
        struct ip_set_net *inst = ip_set_pernet(net);

        /* ip_set_list and the set pointer need to be protected */
        return ip_set_dereference_nfnl(inst->ip_set_list)[index];
}

static inline void
ip_set_lock(struct ip_set *set)
{
        if (!set->variant->region_lock)
                spin_lock_bh(&set->lock);
}

static inline void
ip_set_unlock(struct ip_set *set)
{
        if (!set->variant->region_lock)
                spin_unlock_bh(&set->lock);
}

int
ip_set_test(ip_set_id_t index, const struct sk_buff *skb,
            const struct xt_action_param *par, struct ip_set_adt_opt *opt)
{
        struct ip_set *set = ip_set_rcu_get(xt_net(par), index);
        int ret = 0;

        BUG_ON(!set);
        pr_debug("set %s, index %u\n", set->name, index);

        if (opt->dim < set->type->dimension ||
            !(opt->family == set->family || set->family == NFPROTO_UNSPEC))
                return 0;

        ret = set->variant->kadt(set, skb, par, IPSET_TEST, opt);

        if (ret == -EAGAIN) {
                /* Type requests element to be completed */
                pr_debug("element must be completed, ADD is triggered\n");
                ip_set_lock(set);
                set->variant->kadt(set, skb, par, IPSET_ADD, opt);
                ip_set_unlock(set);
                ret = 1;
        } else {
                /* --return-nomatch: invert matched element */
                if ((opt->cmdflags & IPSET_FLAG_RETURN_NOMATCH) &&
                    (set->type->features & IPSET_TYPE_NOMATCH) &&
                    (ret > 0 || ret == -ENOTEMPTY))
                        ret = -ret;
        }

        /* Convert error codes to nomatch */
        return (ret < 0 ? 0 : ret);
}
EXPORT_SYMBOL_GPL(ip_set_test);

int
ip_set_add(ip_set_id_t index, const struct sk_buff *skb,
           const struct xt_action_param *par, struct ip_set_adt_opt *opt)
{
        struct ip_set *set = ip_set_rcu_get(xt_net(par), index);
        int ret;

        BUG_ON(!set);
        pr_debug("set %s, index %u\n", set->name, index);

        if (opt->dim < set->type->dimension ||
            !(opt->family == set->family || set->family == NFPROTO_UNSPEC))
                return -IPSET_ERR_TYPE_MISMATCH;

        ip_set_lock(set);
        ret = set->variant->kadt(set, skb, par, IPSET_ADD, opt);
        ip_set_unlock(set);

        return ret;
}
EXPORT_SYMBOL_GPL(ip_set_add);

int
ip_set_del(ip_set_id_t index, const struct sk_buff *skb,
           const struct xt_action_param *par, struct ip_set_adt_opt *opt)
{
        struct ip_set *set = ip_set_rcu_get(xt_net(par), index);
        int ret = 0;

        BUG_ON(!set);
        pr_debug("set %s, index %u\n", set->name, index);

        if (opt->dim < set->type->dimension ||
            !(opt->family == set->family || set->family == NFPROTO_UNSPEC))
                return -IPSET_ERR_TYPE_MISMATCH;

        ip_set_lock(set);
        ret = set->variant->kadt(set, skb, par, IPSET_DEL, opt);
        ip_set_unlock(set);

        return ret;
}
EXPORT_SYMBOL_GPL(ip_set_del);

/* Find set by name, reference it once. The reference makes sure the
 * thing pointed to, does not go away under our feet.
 *
 */
ip_set_id_t
ip_set_get_byname(struct net *net, const char *name, struct ip_set **set)
{
        ip_set_id_t i, index = IPSET_INVALID_ID;
        struct ip_set *s;
        struct ip_set_net *inst = ip_set_pernet(net);

        rcu_read_lock();
        for (i = 0; i < inst->ip_set_max; i++) {
                s = rcu_dereference(inst->ip_set_list)[i];
                if (s && STRNCMP(s->name, name)) {
                        __ip_set_get(s);
                        index = i;
                        *set = s;
                        break;
                }
        }
        rcu_read_unlock();

        return index;
}
EXPORT_SYMBOL_GPL(ip_set_get_byname);

/* If the given set pointer points to a valid set, decrement
 * reference count by 1. The caller shall not assume the index
 * to be valid, after calling this function.
 *
 */

static void
__ip_set_put_byindex(struct ip_set_net *inst, ip_set_id_t index)
{
        struct ip_set *set;

        rcu_read_lock();
        set = rcu_dereference(inst->ip_set_list)[index];
        if (set)
                __ip_set_put(set);
        rcu_read_unlock();
}

void
ip_set_put_byindex(struct net *net, ip_set_id_t index)
{
        struct ip_set_net *inst = ip_set_pernet(net);

        __ip_set_put_byindex(inst, index);
}
EXPORT_SYMBOL_GPL(ip_set_put_byindex);

/* Get the name of a set behind a set index.
 * Set itself is protected by RCU, but its name isn't: to protect against
 * renaming, grab ip_set_ref_lock as reader (see ip_set_rename()) and copy the
 * name.
 */
void
ip_set_name_byindex(struct net *net, ip_set_id_t index, char *name)
{
        struct ip_set *set = ip_set_rcu_get(net, index);

        BUG_ON(!set);

        read_lock_bh(&ip_set_ref_lock);
        strscpy_pad(name, set->name, IPSET_MAXNAMELEN);
        read_unlock_bh(&ip_set_ref_lock);
}
EXPORT_SYMBOL_GPL(ip_set_name_byindex);

/* Routines to call by external subsystems, which do not
 * call nfnl_lock for us.
 */

/* Find set by index, reference it once. The reference makes sure the
 * thing pointed to, does not go away under our feet.
 *
 * The nfnl mutex is used in the function.
 */
ip_set_id_t
ip_set_nfnl_get_byindex(struct net *net, ip_set_id_t index)
{
        struct ip_set *set;
        struct ip_set_net *inst = ip_set_pernet(net);

        if (index >= inst->ip_set_max)
                return IPSET_INVALID_ID;

        nfnl_lock(NFNL_SUBSYS_IPSET);
        set = ip_set(inst, index);
        if (set)
                __ip_set_get(set);
        else
                index = IPSET_INVALID_ID;
        nfnl_unlock(NFNL_SUBSYS_IPSET);

        return index;
}
EXPORT_SYMBOL_GPL(ip_set_nfnl_get_byindex);

/* If the given set pointer points to a valid set, decrement
 * reference count by 1. The caller shall not assume the index
 * to be valid, after calling this function.
 *
 * The nfnl mutex is used in the function.
 */
void
ip_set_nfnl_put(struct net *net, ip_set_id_t index)
{
        struct ip_set *set;
        struct ip_set_net *inst = ip_set_pernet(net);

        nfnl_lock(NFNL_SUBSYS_IPSET);
        if (!inst->is_deleted) { /* already deleted from ip_set_net_exit() */
                set = ip_set(inst, index);
                if (set)
                        __ip_set_put(set);
        }
        nfnl_unlock(NFNL_SUBSYS_IPSET);
}
EXPORT_SYMBOL_GPL(ip_set_nfnl_put);

/* Communication protocol with userspace over netlink.
 *
 * The commands are serialized by the nfnl mutex.
 */

static inline u8 protocol(const struct nlattr * const tb[])
{
        return nla_get_u8(tb[IPSET_ATTR_PROTOCOL]);
}

static inline bool
protocol_failed(const struct nlattr * const tb[])
{
        return !tb[IPSET_ATTR_PROTOCOL] || protocol(tb) != IPSET_PROTOCOL;
}

static inline bool
protocol_min_failed(const struct nlattr * const tb[])
{
        return !tb[IPSET_ATTR_PROTOCOL] || protocol(tb) < IPSET_PROTOCOL_MIN;
}

static inline u32
flag_exist(const struct nlmsghdr *nlh)
{
        return nlh->nlmsg_flags & NLM_F_EXCL ? 0 : IPSET_FLAG_EXIST;
}

static struct nlmsghdr *
start_msg(struct sk_buff *skb, u32 portid, u32 seq, unsigned int flags,
          enum ipset_cmd cmd)
{
        return nfnl_msg_put(skb, portid, seq,
                            nfnl_msg_type(NFNL_SUBSYS_IPSET, cmd), flags,
                            NFPROTO_IPV4, NFNETLINK_V0, 0);
}

/* Create a set */

static const struct nla_policy ip_set_create_policy[IPSET_ATTR_CMD_MAX + 1] = {
        [IPSET_ATTR_PROTOCOL]        = { .type = NLA_U8 },
        [IPSET_ATTR_SETNAME]        = { .type = NLA_NUL_STRING,
                                    .len = IPSET_MAXNAMELEN - 1 },
        [IPSET_ATTR_TYPENAME]        = { .type = NLA_NUL_STRING,
                                    .len = IPSET_MAXNAMELEN - 1},
        [IPSET_ATTR_REVISION]        = { .type = NLA_U8 },
        [IPSET_ATTR_FAMILY]        = { .type = NLA_U8 },
        [IPSET_ATTR_DATA]        = { .type = NLA_NESTED },
};

static struct ip_set *
find_set_and_id(struct ip_set_net *inst, const char *name, ip_set_id_t *id)
{
        struct ip_set *set = NULL;
        ip_set_id_t i;

        *id = IPSET_INVALID_ID;
        for (i = 0; i < inst->ip_set_max; i++) {
                set = ip_set(inst, i);
                if (set && STRNCMP(set->name, name)) {
                        *id = i;
                        break;
                }
        }
        return (*id == IPSET_INVALID_ID ? NULL : set);
}

static inline struct ip_set *
find_set(struct ip_set_net *inst, const char *name)
{
        ip_set_id_t id;

        return find_set_and_id(inst, name, &id);
}

static int
find_free_id(struct ip_set_net *inst, const char *name, ip_set_id_t *index,
             struct ip_set **set)
{
        struct ip_set *s;
        ip_set_id_t i;

        *index = IPSET_INVALID_ID;
        for (i = 0;  i < inst->ip_set_max; i++) {
                s = ip_set(inst, i);
                if (!s) {
                        if (*index == IPSET_INVALID_ID)
                                *index = i;
                } else if (STRNCMP(name, s->name)) {
                        /* Name clash */
                        *set = s;
                        return -EEXIST;
                }
        }
        if (*index == IPSET_INVALID_ID)
                /* No free slot remained */
                return -IPSET_ERR_MAX_SETS;
        return 0;
}

static int ip_set_none(struct sk_buff *skb, const struct nfnl_info *info,
                       const struct nlattr * const attr[])
{
        return -EOPNOTSUPP;
}

static int ip_set_create(struct sk_buff *skb, const struct nfnl_info *info,
                         const struct nlattr * const attr[])
{
        struct ip_set_net *inst = ip_set_pernet(info->net);
        struct ip_set *set, *clash = NULL;
        ip_set_id_t index = IPSET_INVALID_ID;
        struct nlattr *tb[IPSET_ATTR_CREATE_MAX + 1] = {};
        const char *name, *typename;
        u8 family, revision;
        u32 flags = flag_exist(info->nlh);
        int ret = 0;

        if (unlikely(protocol_min_failed(attr) ||
                     !attr[IPSET_ATTR_SETNAME] ||
                     !attr[IPSET_ATTR_TYPENAME] ||
                     !attr[IPSET_ATTR_REVISION] ||
                     !attr[IPSET_ATTR_FAMILY] ||
                     (attr[IPSET_ATTR_DATA] &&
                      !flag_nested(attr[IPSET_ATTR_DATA]))))
                return -IPSET_ERR_PROTOCOL;

        name = nla_data(attr[IPSET_ATTR_SETNAME]);
        typename = nla_data(attr[IPSET_ATTR_TYPENAME]);
        family = nla_get_u8(attr[IPSET_ATTR_FAMILY]);
        revision = nla_get_u8(attr[IPSET_ATTR_REVISION]);
        pr_debug("setname: %s, typename: %s, family: %s, revision: %u\n",
                 name, typename, family_name(family), revision);

        /* First, and without any locks, allocate and initialize
         * a normal base set structure.
         */
        set = kzalloc(sizeof(*set), GFP_KERNEL);
        if (!set)
                return -ENOMEM;
        spin_lock_init(&set->lock);
        strscpy(set->name, name, IPSET_MAXNAMELEN);
        set->family = family;
        set->revision = revision;

        /* Next, check that we know the type, and take
         * a reference on the type, to make sure it stays available
         * while constructing our new set.
         *
         * After referencing the type, we try to create the type
         * specific part of the set without holding any locks.
         */
        ret = find_set_type_get(typename, family, revision, &set->type);
        if (ret)
                goto out;

        /* Without holding any locks, create private part. */
        if (attr[IPSET_ATTR_DATA] &&
            nla_parse_nested(tb, IPSET_ATTR_CREATE_MAX, attr[IPSET_ATTR_DATA],
                             set->type->create_policy, NULL)) {
                ret = -IPSET_ERR_PROTOCOL;
                goto put_out;
        }
        /* Set create flags depending on the type revision */
        set->flags |= set->type->create_flags[revision];

        ret = set->type->create(info->net, set, tb, flags);
        if (ret != 0)
                goto put_out;

        /* BTW, ret==0 here. */

        /* Here, we have a valid, constructed set and we are protected
         * by the nfnl mutex. Find the first free index in ip_set_list
         * and check clashing.
         */
        ret = find_free_id(inst, set->name, &index, &clash);
        if (ret == -EEXIST) {
                /* If this is the same set and requested, ignore error */
                if ((flags & IPSET_FLAG_EXIST) &&
                    STRNCMP(set->type->name, clash->type->name) &&
                    set->type->family == clash->type->family &&
                    set->type->revision_min == clash->type->revision_min &&
                    set->type->revision_max == clash->type->revision_max &&
                    set->variant->same_set(set, clash))
                        ret = 0;
                goto cleanup;
        } else if (ret == -IPSET_ERR_MAX_SETS) {
                struct ip_set **list, **tmp;
                ip_set_id_t i = inst->ip_set_max + IP_SET_INC;

                if (i < inst->ip_set_max || i == IPSET_INVALID_ID)
                        /* Wraparound */
                        goto cleanup;

                list = kvcalloc(i, sizeof(struct ip_set *), GFP_KERNEL);
                if (!list)
                        goto cleanup;
                /* nfnl mutex is held, both lists are valid */
                tmp = ip_set_dereference(inst);
                memcpy(list, tmp, sizeof(struct ip_set *) * inst->ip_set_max);
                rcu_assign_pointer(inst->ip_set_list, list);
                /* Make sure all current packets have passed through */
                synchronize_net();
                /* Use new list */
                index = inst->ip_set_max;
                inst->ip_set_max = i;
                kvfree(tmp);
                ret = 0;
        } else if (ret) {
                goto cleanup;
        }

        /* Finally! Add our shiny new set to the list, and be done. */
        pr_debug("create: '%s' created with index %u!\n", set->name, index);
        ip_set(inst, index) = set;

        return ret;

cleanup:
        set->variant->cancel_gc(set);
        set->variant->destroy(set);
put_out:
        module_put(set->type->me);
out:
        kfree(set);
        return ret;
}

/* Destroy sets */

static const struct nla_policy
ip_set_setname_policy[IPSET_ATTR_CMD_MAX + 1] = {
        [IPSET_ATTR_PROTOCOL]        = { .type = NLA_U8 },
        [IPSET_ATTR_SETNAME]        = { .type = NLA_NUL_STRING,
                                    .len = IPSET_MAXNAMELEN - 1 },
};

/* In order to return quickly when destroying a single set, it is split
 * into two stages:
 * - Cancel garbage collector
 * - Destroy the set itself via call_rcu()
 */

static void
ip_set_destroy_set_rcu(struct rcu_head *head)
{
        struct ip_set *set = container_of(head, struct ip_set, rcu);

        set->variant->destroy(set);
        module_put(set->type->me);
        kfree(set);
}

static void
_destroy_all_sets(struct ip_set_net *inst)
{
        struct ip_set *set;
        ip_set_id_t i;
        bool need_wait = false;

        /* First cancel gc's: set:list sets are flushed as well */
        for (i = 0; i < inst->ip_set_max; i++) {
                set = ip_set(inst, i);
                if (set) {
                        set->variant->cancel_gc(set);
                        if (set->type->features & IPSET_TYPE_NAME)
                                need_wait = true;
                }
        }
        /* Must wait for flush to be really finished  */
        if (need_wait)
                rcu_barrier();
        for (i = 0; i < inst->ip_set_max; i++) {
                set = ip_set(inst, i);
                if (set) {
                        ip_set(inst, i) = NULL;
                        set->variant->destroy(set);
                        module_put(set->type->me);
                        kfree(set);
                }
        }
}

static int ip_set_destroy(struct sk_buff *skb, const struct nfnl_info *info,
                          const struct nlattr * const attr[])
{
        struct ip_set_net *inst = ip_set_pernet(info->net);
        struct ip_set *s;
        ip_set_id_t i;
        int ret = 0;

        if (unlikely(protocol_min_failed(attr)))
                return -IPSET_ERR_PROTOCOL;

        /* Commands are serialized and references are
         * protected by the ip_set_ref_lock.
         * External systems (i.e. xt_set) must call
         * ip_set_nfnl_get_* functions, that way we
         * can safely check references here.
         *
         * list:set timer can only decrement the reference
         * counter, so if it's already zero, we can proceed
         * without holding the lock.
         */
        if (!attr[IPSET_ATTR_SETNAME]) {
                read_lock_bh(&ip_set_ref_lock);
                for (i = 0; i < inst->ip_set_max; i++) {
                        s = ip_set(inst, i);
                        if (s && (s->ref || s->ref_netlink)) {
                                ret = -IPSET_ERR_BUSY;
                                goto out;
                        }
                }
                inst->is_destroyed = true;
                read_unlock_bh(&ip_set_ref_lock);
                _destroy_all_sets(inst);
                /* Modified by ip_set_destroy() only, which is serialized */
                inst->is_destroyed = false;
        } else {
                u32 flags = flag_exist(info->nlh);
                u16 features = 0;

                read_lock_bh(&ip_set_ref_lock);
                s = find_set_and_id(inst, nla_data(attr[IPSET_ATTR_SETNAME]),
                                    &i);
                if (!s) {
                        if (!(flags & IPSET_FLAG_EXIST))
                                ret = -ENOENT;
                        goto out;
                } else if (s->ref || s->ref_netlink) {
                        ret = -IPSET_ERR_BUSY;
                        goto out;
                }
                features = s->type->features;
                ip_set(inst, i) = NULL;
                read_unlock_bh(&ip_set_ref_lock);
                /* Must cancel garbage collectors */
                s->variant->cancel_gc(s);
                if (features & IPSET_TYPE_NAME) {
                        /* Must wait for flush to be really finished  */
                        rcu_barrier();
                }
                call_rcu(&s->rcu, ip_set_destroy_set_rcu);
        }
        return 0;
out:
        read_unlock_bh(&ip_set_ref_lock);
        return ret;
}

/* Flush sets */

static void
ip_set_flush_set(struct ip_set *set)
{
        pr_debug("set: %s\n",  set->name);

        ip_set_lock(set);
        set->variant->flush(set);
        ip_set_unlock(set);
}

static int ip_set_flush(struct sk_buff *skb, const struct nfnl_info *info,
                        const struct nlattr * const attr[])
{
        struct ip_set_net *inst = ip_set_pernet(info->net);
        struct ip_set *s;
        ip_set_id_t i;

        if (unlikely(protocol_min_failed(attr)))
                return -IPSET_ERR_PROTOCOL;

        if (!attr[IPSET_ATTR_SETNAME]) {
                for (i = 0; i < inst->ip_set_max; i++) {
                        s = ip_set(inst, i);
                        if (s)
                                ip_set_flush_set(s);
                }
        } else {
                s = find_set(inst, nla_data(attr[IPSET_ATTR_SETNAME]));
                if (!s)
                        return -ENOENT;

                ip_set_flush_set(s);
        }

        return 0;
}

/* Rename a set */

static const struct nla_policy
ip_set_setname2_policy[IPSET_ATTR_CMD_MAX + 1] = {
        [IPSET_ATTR_PROTOCOL]        = { .type = NLA_U8 },
        [IPSET_ATTR_SETNAME]        = { .type = NLA_NUL_STRING,
                                    .len = IPSET_MAXNAMELEN - 1 },
        [IPSET_ATTR_SETNAME2]        = { .type = NLA_NUL_STRING,
                                    .len = IPSET_MAXNAMELEN - 1 },
};

static int ip_set_rename(struct sk_buff *skb, const struct nfnl_info *info,
                         const struct nlattr * const attr[])
{
        struct ip_set_net *inst = ip_set_pernet(info->net);
        struct ip_set *set, *s;
        const char *name2;
        ip_set_id_t i;
        int ret = 0;

        if (unlikely(protocol_min_failed(attr) ||
                     !attr[IPSET_ATTR_SETNAME] ||
                     !attr[IPSET_ATTR_SETNAME2]))
                return -IPSET_ERR_PROTOCOL;

        set = find_set(inst, nla_data(attr[IPSET_ATTR_SETNAME]));
        if (!set)
                return -ENOENT;

        write_lock_bh(&ip_set_ref_lock);
        if (set->ref != 0 || set->ref_netlink != 0) {
                ret = -IPSET_ERR_REFERENCED;
                goto out;
        }

        name2 = nla_data(attr[IPSET_ATTR_SETNAME2]);
        for (i = 0; i < inst->ip_set_max; i++) {
                s = ip_set(inst, i);
                if (s && STRNCMP(s->name, name2)) {
                        ret = -IPSET_ERR_EXIST_SETNAME2;
                        goto out;
                }
        }
        strscpy_pad(set->name, name2, IPSET_MAXNAMELEN);

out:
        write_unlock_bh(&ip_set_ref_lock);
        return ret;
}

/* Swap two sets so that name/index points to the other.
 * References and set names are also swapped.
 *
 * The commands are serialized by the nfnl mutex and references are
 * protected by the ip_set_ref_lock. The kernel interfaces
 * do not hold the mutex but the pointer settings are atomic
 * so the ip_set_list always contains valid pointers to the sets.
 */

static int ip_set_swap(struct sk_buff *skb, const struct nfnl_info *info,
                       const struct nlattr * const attr[])
{
        struct ip_set_net *inst = ip_set_pernet(info->net);
        struct ip_set *from, *to;
        ip_set_id_t from_id, to_id;
        char from_name[IPSET_MAXNAMELEN];

        if (unlikely(protocol_min_failed(attr) ||
                     !attr[IPSET_ATTR_SETNAME] ||
                     !attr[IPSET_ATTR_SETNAME2]))
                return -IPSET_ERR_PROTOCOL;

        from = find_set_and_id(inst, nla_data(attr[IPSET_ATTR_SETNAME]),
                               &from_id);
        if (!from)
                return -ENOENT;

        to = find_set_and_id(inst, nla_data(attr[IPSET_ATTR_SETNAME2]),
                             &to_id);
        if (!to)
                return -IPSET_ERR_EXIST_SETNAME2;

        /* Features must not change.
         * Not an artifical restriction anymore, as we must prevent
         * possible loops created by swapping in setlist type of sets.
         */
        if (!(from->type->features == to->type->features &&
              from->family == to->family))
                return -IPSET_ERR_TYPE_MISMATCH;

        write_lock_bh(&ip_set_ref_lock);

        if (from->ref_netlink || to->ref_netlink) {
                write_unlock_bh(&ip_set_ref_lock);
                return -EBUSY;
        }

        strscpy_pad(from_name, from->name, IPSET_MAXNAMELEN);
        strscpy_pad(from->name, to->name, IPSET_MAXNAMELEN);
        strscpy_pad(to->name, from_name, IPSET_MAXNAMELEN);

        swap(from->ref, to->ref);
        ip_set(inst, from_id) = to;
        ip_set(inst, to_id) = from;
        write_unlock_bh(&ip_set_ref_lock);

        return 0;
}

/* List/save set data */

#define DUMP_INIT        0
#define DUMP_ALL        1
#define DUMP_ONE        2
#define DUMP_LAST        3

#define DUMP_TYPE(arg)                (((u32)(arg)) & 0x0000FFFF)
#define DUMP_FLAGS(arg)                (((u32)(arg)) >> 16)

int
ip_set_put_flags(struct sk_buff *skb, struct ip_set *set)
{
        u32 cadt_flags = 0;

        if (SET_WITH_TIMEOUT(set))
                if (unlikely(nla_put_net32(skb, IPSET_ATTR_TIMEOUT,
                                           htonl(set->timeout))))
                        return -EMSGSIZE;
        if (SET_WITH_COUNTER(set))
                cadt_flags |= IPSET_FLAG_WITH_COUNTERS;
        if (SET_WITH_COMMENT(set))
                cadt_flags |= IPSET_FLAG_WITH_COMMENT;
        if (SET_WITH_SKBINFO(set))
                cadt_flags |= IPSET_FLAG_WITH_SKBINFO;
        if (SET_WITH_FORCEADD(set))
                cadt_flags |= IPSET_FLAG_WITH_FORCEADD;

        if (!cadt_flags)
                return 0;
        return nla_put_net32(skb, IPSET_ATTR_CADT_FLAGS, htonl(cadt_flags));
}
EXPORT_SYMBOL_GPL(ip_set_put_flags);

static int
ip_set_dump_done(struct netlink_callback *cb)
{
        if (cb->args[IPSET_CB_ARG0]) {
                struct ip_set_net *inst =
                        (struct ip_set_net *)cb->args[IPSET_CB_NET];
                ip_set_id_t index = (ip_set_id_t)cb->args[IPSET_CB_INDEX];
                struct ip_set *set = ip_set_ref_netlink(inst, index);

                if (set->variant->uref)
                        set->variant->uref(set, cb, false);
                pr_debug("release set %s\n", set->name);
                __ip_set_put_netlink(set);
        }
        return 0;
}

static inline void
dump_attrs(struct nlmsghdr *nlh)
{
        const struct nlattr *attr;
        int rem;

        pr_debug("dump nlmsg\n");
        nlmsg_for_each_attr(attr, nlh, sizeof(struct nfgenmsg), rem) {
                pr_debug("type: %u, len %u\n", nla_type(attr), attr->nla_len);
        }
}

static const struct nla_policy
ip_set_dump_policy[IPSET_ATTR_CMD_MAX + 1] = {
        [IPSET_ATTR_PROTOCOL]        = { .type = NLA_U8 },
        [IPSET_ATTR_SETNAME]        = { .type = NLA_NUL_STRING,
                                    .len = IPSET_MAXNAMELEN - 1 },
        [IPSET_ATTR_FLAGS]        = { .type = NLA_U32 },
};

static int
ip_set_dump_start(struct netlink_callback *cb)
{
        struct nlmsghdr *nlh = nlmsg_hdr(cb->skb);
        int min_len = nlmsg_total_size(sizeof(struct nfgenmsg));
        struct nlattr *cda[IPSET_ATTR_CMD_MAX + 1];
        struct nlattr *attr = (void *)nlh + min_len;
        struct sk_buff *skb = cb->skb;
        struct ip_set_net *inst = ip_set_pernet(sock_net(skb->sk));
        u32 dump_type;
        int ret;

        ret = nla_parse(cda, IPSET_ATTR_CMD_MAX, attr,
                        nlh->nlmsg_len - min_len,
                        ip_set_dump_policy, NULL);
        if (ret)
                goto error;

        cb->args[IPSET_CB_PROTO] = nla_get_u8(cda[IPSET_ATTR_PROTOCOL]);
        if (cda[IPSET_ATTR_SETNAME]) {
                ip_set_id_t index;
                struct ip_set *set;

                set = find_set_and_id(inst, nla_data(cda[IPSET_ATTR_SETNAME]),
                                      &index);
                if (!set) {
                        ret = -ENOENT;
                        goto error;
                }
                dump_type = DUMP_ONE;
                cb->args[IPSET_CB_INDEX] = index;
        } else {
                dump_type = DUMP_ALL;
        }

        if (cda[IPSET_ATTR_FLAGS]) {
                u32 f = ip_set_get_h32(cda[IPSET_ATTR_FLAGS]);

                dump_type |= (f << 16);
        }
        cb->args[IPSET_CB_NET] = (unsigned long)inst;
        cb->args[IPSET_CB_DUMP] = dump_type;

        return 0;

error:
        /* We have to create and send the error message manually :-( */
        if (nlh->nlmsg_flags & NLM_F_ACK) {
                netlink_ack(cb->skb, nlh, ret, NULL);
        }
        return ret;
}

static int
ip_set_dump_do(struct sk_buff *skb, struct netlink_callback *cb)
{
        ip_set_id_t index = IPSET_INVALID_ID, max;
        struct ip_set *set = NULL;
        struct nlmsghdr *nlh = NULL;
        unsigned int flags = NETLINK_CB(cb->skb).portid ? NLM_F_MULTI : 0;
        struct ip_set_net *inst = ip_set_pernet(sock_net(skb->sk));
        u32 dump_type, dump_flags;
        bool is_destroyed;
        int ret = 0;

        if (!cb->args[IPSET_CB_DUMP])
                return -EINVAL;

        if (cb->args[IPSET_CB_INDEX] >= inst->ip_set_max)
                goto out;

        dump_type = DUMP_TYPE(cb->args[IPSET_CB_DUMP]);
        dump_flags = DUMP_FLAGS(cb->args[IPSET_CB_DUMP]);
        max = dump_type == DUMP_ONE ? cb->args[IPSET_CB_INDEX] + 1
                                    : inst->ip_set_max;
dump_last:
        pr_debug("dump type, flag: %u %u index: %ld\n",
                 dump_type, dump_flags, cb->args[IPSET_CB_INDEX]);
        for (; cb->args[IPSET_CB_INDEX] < max; cb->args[IPSET_CB_INDEX]++) {
                index = (ip_set_id_t)cb->args[IPSET_CB_INDEX];
                write_lock_bh(&ip_set_ref_lock);
                set = ip_set(inst, index);
                is_destroyed = inst->is_destroyed;
                if (!set || is_destroyed) {
                        write_unlock_bh(&ip_set_ref_lock);
                        if (dump_type == DUMP_ONE) {
                                ret = -ENOENT;
                                goto out;
                        }
                        if (is_destroyed) {
                                /* All sets are just being destroyed */
                                ret = 0;
                                goto out;
                        }
                        continue;
                }
                /* When dumping all sets, we must dump "sorted"
                 * so that lists (unions of sets) are dumped last.
                 */
                if (dump_type != DUMP_ONE &&
                    ((dump_type == DUMP_ALL) ==
                     !!(set->type->features & IPSET_DUMP_LAST))) {
                        write_unlock_bh(&ip_set_ref_lock);
                        continue;
                }
                pr_debug("List set: %s\n", set->name);
                if (!cb->args[IPSET_CB_ARG0]) {
                        /* Start listing: make sure set won't be destroyed */
                        pr_debug("reference set\n");
                        set->ref_netlink++;
                }
                write_unlock_bh(&ip_set_ref_lock);
                nlh = start_msg(skb, NETLINK_CB(cb->skb).portid,
                                cb->nlh->nlmsg_seq, flags,
                                IPSET_CMD_LIST);
                if (!nlh) {
                        ret = -EMSGSIZE;
                        goto release_refcount;
                }
                if (nla_put_u8(skb, IPSET_ATTR_PROTOCOL,
                               cb->args[IPSET_CB_PROTO]) ||
                    nla_put_string(skb, IPSET_ATTR_SETNAME, set->name))
                        goto nla_put_failure;
                if (dump_flags & IPSET_FLAG_LIST_SETNAME)
                        goto next_set;
                switch (cb->args[IPSET_CB_ARG0]) {
                case 0:
                        /* Core header data */
                        if (nla_put_string(skb, IPSET_ATTR_TYPENAME,
                                           set->type->name) ||
                            nla_put_u8(skb, IPSET_ATTR_FAMILY,
                                       set->family) ||
                            nla_put_u8(skb, IPSET_ATTR_REVISION,
                                       set->revision))
                                goto nla_put_failure;
                        if (cb->args[IPSET_CB_PROTO] > IPSET_PROTOCOL_MIN &&
                            nla_put_net16(skb, IPSET_ATTR_INDEX, htons(index)))
                                goto nla_put_failure;
                        ret = set->variant->head(set, skb);
                        if (ret < 0)
                                goto release_refcount;
                        if (dump_flags & IPSET_FLAG_LIST_HEADER)
                                goto next_set;
                        if (set->variant->uref)
                                set->variant->uref(set, cb, true);
                        fallthrough;
                default:
                        ret = set->variant->list(set, skb, cb);
                        if (!cb->args[IPSET_CB_ARG0])
                                /* Set is done, proceed with next one */
                                goto next_set;
                        goto release_refcount;
                }
        }
        /* If we dump all sets, continue with dumping last ones */
        if (dump_type == DUMP_ALL) {
                dump_type = DUMP_LAST;
                cb->args[IPSET_CB_DUMP] = dump_type | (dump_flags << 16);
                cb->args[IPSET_CB_INDEX] = 0;
                if (set && set->variant->uref)
                        set->variant->uref(set, cb, false);
                goto dump_last;
        }
        goto out;

nla_put_failure:
        ret = -EFAULT;
next_set:
        if (dump_type == DUMP_ONE)
                cb->args[IPSET_CB_INDEX] = IPSET_INVALID_ID;
        else
                cb->args[IPSET_CB_INDEX]++;
release_refcount:
        /* If there was an error or set is done, release set */
        if (ret || !cb->args[IPSET_CB_ARG0]) {
                set = ip_set_ref_netlink(inst, index);
                if (set->variant->uref)
                        set->variant->uref(set, cb, false);
                pr_debug("release set %s\n", set->name);
                __ip_set_put_netlink(set);
                cb->args[IPSET_CB_ARG0] = 0;
        }
out:
        if (nlh) {
                nlmsg_end(skb, nlh);
                pr_debug("nlmsg_len: %u\n", nlh->nlmsg_len);
                dump_attrs(nlh);
        }

        return ret < 0 ? ret : skb->len;
}

static int ip_set_dump(struct sk_buff *skb, const struct nfnl_info *info,
                       const struct nlattr * const attr[])
{
        if (unlikely(protocol_min_failed(attr)))
                return -IPSET_ERR_PROTOCOL;

        {
                struct netlink_dump_control c = {
                        .start = ip_set_dump_start,
                        .dump = ip_set_dump_do,
                        .done = ip_set_dump_done,
                };
                return netlink_dump_start(info->sk, skb, info->nlh, &c);
        }
}

/* Add, del and test */

static const struct nla_policy ip_set_adt_policy[IPSET_ATTR_CMD_MAX + 1] = {
        [IPSET_ATTR_PROTOCOL]        = { .type = NLA_U8 },
        [IPSET_ATTR_SETNAME]        = { .type = NLA_NUL_STRING,
                                    .len = IPSET_MAXNAMELEN - 1 },
        [IPSET_ATTR_LINENO]        = { .type = NLA_U32 },
        [IPSET_ATTR_DATA]        = { .type = NLA_NESTED },
        [IPSET_ATTR_ADT]        = { .type = NLA_NESTED },
};

static int
call_ad(struct net *net, struct sock *ctnl, struct sk_buff *skb,
        struct ip_set *set, struct nlattr *tb[], enum ipset_adt adt,
        u32 flags, bool use_lineno)
{
        int ret;
        u32 lineno = 0;
        bool eexist = flags & IPSET_FLAG_EXIST, retried = false;

        do {
                if (retried) {
                        __ip_set_get_netlink(set);
                        nfnl_unlock(NFNL_SUBSYS_IPSET);
                        cond_resched();
                        nfnl_lock(NFNL_SUBSYS_IPSET);
                        __ip_set_put_netlink(set);
                }

                ip_set_lock(set);
                ret = set->variant->uadt(set, tb, adt, &lineno, flags, retried);
                ip_set_unlock(set);
                retried = true;
        } while (ret == -ERANGE ||
                 (ret == -EAGAIN &&
                  set->variant->resize &&
                  (ret = set->variant->resize(set, retried)) == 0));

        if (!ret || (ret == -IPSET_ERR_EXIST && eexist))
                return 0;
        if (lineno && use_lineno) {
                /* Error in restore/batch mode: send back lineno */
                struct nlmsghdr *rep, *nlh = nlmsg_hdr(skb);
                struct sk_buff *skb2;
                struct nlmsgerr *errmsg;
                size_t payload = min(SIZE_MAX,
                                     sizeof(*errmsg) + nlmsg_len(nlh));
                int min_len = nlmsg_total_size(sizeof(struct nfgenmsg));
                struct nlattr *cda[IPSET_ATTR_CMD_MAX + 1];
                struct nlattr *cmdattr;
                u32 *errline;

                skb2 = nlmsg_new(payload, GFP_KERNEL);
                if (!skb2)
                        return -ENOMEM;
                rep = nlmsg_put(skb2, NETLINK_CB(skb).portid,
                                nlh->nlmsg_seq, NLMSG_ERROR, payload, 0);
                errmsg = nlmsg_data(rep);
                errmsg->error = ret;
                unsafe_memcpy(&errmsg->msg, nlh, nlh->nlmsg_len,
                              /* Bounds checked by the skb layer. */);

                cmdattr = (void *)&errmsg->msg + min_len;

                ret = nla_parse(cda, IPSET_ATTR_CMD_MAX, cmdattr,
                                nlh->nlmsg_len - min_len, ip_set_adt_policy,
                                NULL);

                if (ret) {
                        nlmsg_free(skb2);
                        return ret;
                }
                errline = nla_data(cda[IPSET_ATTR_LINENO]);

                *errline = lineno;

                nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
                /* Signal netlink not to send its ACK/errmsg.  */
                return -EINTR;
        }

        return ret;
}

static int ip_set_ad(struct net *net, struct sock *ctnl,
                     struct sk_buff *skb,
                     enum ipset_adt adt,
                     const struct nlmsghdr *nlh,
                     const struct nlattr * const attr[],
                     struct netlink_ext_ack *extack)
{
        struct ip_set_net *inst = ip_set_pernet(net);
        struct ip_set *set;
        struct nlattr *tb[IPSET_ATTR_ADT_MAX + 1] = {};
        const struct nlattr *nla;
        u32 flags = flag_exist(nlh);
        bool use_lineno;
        int ret = 0;

        if (unlikely(protocol_min_failed(attr) ||
                     !attr[IPSET_ATTR_SETNAME] ||
                     !((attr[IPSET_ATTR_DATA] != NULL) ^
                       (attr[IPSET_ATTR_ADT] != NULL)) ||
                     (attr[IPSET_ATTR_DATA] &&
                      !flag_nested(attr[IPSET_ATTR_DATA])) ||
                     (attr[IPSET_ATTR_ADT] &&
                      (!flag_nested(attr[IPSET_ATTR_ADT]) ||
                       !attr[IPSET_ATTR_LINENO]))))
                return -IPSET_ERR_PROTOCOL;

        set = find_set(inst, nla_data(attr[IPSET_ATTR_SETNAME]));
        if (!set)
                return -ENOENT;

        use_lineno = !!attr[IPSET_ATTR_LINENO];
        if (attr[IPSET_ATTR_DATA]) {
                if (nla_parse_nested(tb, IPSET_ATTR_ADT_MAX,
                                     attr[IPSET_ATTR_DATA],
                                     set->type->adt_policy, NULL))
                        return -IPSET_ERR_PROTOCOL;
                ret = call_ad(net, ctnl, skb, set, tb, adt, flags,
                              use_lineno);
        } else {
                int nla_rem;

                nla_for_each_nested(nla, attr[IPSET_ATTR_ADT], nla_rem) {
                        if (nla_type(nla) != IPSET_ATTR_DATA ||
                            !flag_nested(nla) ||
                            nla_parse_nested(tb, IPSET_ATTR_ADT_MAX, nla,
                                             set->type->adt_policy, NULL))
                                return -IPSET_ERR_PROTOCOL;
                        ret = call_ad(net, ctnl, skb, set, tb, adt,
                                      flags, use_lineno);
                        if (ret < 0)
                                return ret;
                }
        }
        return ret;
}

static int ip_set_uadd(struct sk_buff *skb, const struct nfnl_info *info,
                       const struct nlattr * const attr[])
{
        return ip_set_ad(info->net, info->sk, skb,
                         IPSET_ADD, info->nlh, attr, info->extack);
}

static int ip_set_udel(struct sk_buff *skb, const struct nfnl_info *info,
                       const struct nlattr * const attr[])
{
        return ip_set_ad(info->net, info->sk, skb,
                         IPSET_DEL, info->nlh, attr, info->extack);
}

static int ip_set_utest(struct sk_buff *skb, const struct nfnl_info *info,
                        const struct nlattr * const attr[])
{
        struct ip_set_net *inst = ip_set_pernet(info->net);
        struct ip_set *set;
        struct nlattr *tb[IPSET_ATTR_ADT_MAX + 1] = {};
        int ret = 0;
        u32 lineno;

        if (unlikely(protocol_min_failed(attr) ||
                     !attr[IPSET_ATTR_SETNAME] ||
                     !attr[IPSET_ATTR_DATA] ||
                     !flag_nested(attr[IPSET_ATTR_DATA])))
                return -IPSET_ERR_PROTOCOL;

        set = find_set(inst, nla_data(attr[IPSET_ATTR_SETNAME]));
        if (!set)
                return -ENOENT;

        if (nla_parse_nested(tb, IPSET_ATTR_ADT_MAX, attr[IPSET_ATTR_DATA],
                             set->type->adt_policy, NULL))
                return -IPSET_ERR_PROTOCOL;

        rcu_read_lock_bh();
        ret = set->variant->uadt(set, tb, IPSET_TEST, &lineno, 0, 0);
        rcu_read_unlock_bh();
        /* Userspace can't trigger element to be re-added */
        if (ret == -EAGAIN)
                ret = 1;

        return ret > 0 ? 0 : -IPSET_ERR_EXIST;
}

/* Get headed data of a set */

static int ip_set_header(struct sk_buff *skb, const struct nfnl_info *info,
                         const struct nlattr * const attr[])
{
        struct ip_set_net *inst = ip_set_pernet(info->net);
        const struct ip_set *set;
        struct sk_buff *skb2;
        struct nlmsghdr *nlh2;

        if (unlikely(protocol_min_failed(attr) ||
                     !attr[IPSET_ATTR_SETNAME]))
                return -IPSET_ERR_PROTOCOL;

        set = find_set(inst, nla_data(attr[IPSET_ATTR_SETNAME]));
        if (!set)
                return -ENOENT;

        skb2 = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!skb2)
                return -ENOMEM;

        nlh2 = start_msg(skb2, NETLINK_CB(skb).portid, info->nlh->nlmsg_seq, 0,
                         IPSET_CMD_HEADER);
        if (!nlh2)
                goto nlmsg_failure;
        if (nla_put_u8(skb2, IPSET_ATTR_PROTOCOL, protocol(attr)) ||
            nla_put_string(skb2, IPSET_ATTR_SETNAME, set->name) ||
            nla_put_string(skb2, IPSET_ATTR_TYPENAME, set->type->name) ||
            nla_put_u8(skb2, IPSET_ATTR_FAMILY, set->family) ||
            nla_put_u8(skb2, IPSET_ATTR_REVISION, set->revision))
                goto nla_put_failure;
        nlmsg_end(skb2, nlh2);

        return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);

nla_put_failure:
        nlmsg_cancel(skb2, nlh2);
nlmsg_failure:
        kfree_skb(skb2);
        return -EMSGSIZE;
}

/* Get type data */

static const struct nla_policy ip_set_type_policy[IPSET_ATTR_CMD_MAX + 1] = {
        [IPSET_ATTR_PROTOCOL]        = { .type = NLA_U8 },
        [IPSET_ATTR_TYPENAME]        = { .type = NLA_NUL_STRING,
                                    .len = IPSET_MAXNAMELEN - 1 },
        [IPSET_ATTR_FAMILY]        = { .type = NLA_U8 },
};

static int ip_set_type(struct sk_buff *skb, const struct nfnl_info *info,
                       const struct nlattr * const attr[])
{
        struct sk_buff *skb2;
        struct nlmsghdr *nlh2;
        u8 family, min, max;
        const char *typename;
        int ret = 0;

        if (unlikely(protocol_min_failed(attr) ||
                     !attr[IPSET_ATTR_TYPENAME] ||
                     !attr[IPSET_ATTR_FAMILY]))
                return -IPSET_ERR_PROTOCOL;

        family = nla_get_u8(attr[IPSET_ATTR_FAMILY]);
        typename = nla_data(attr[IPSET_ATTR_TYPENAME]);
        ret = find_set_type_minmax(typename, family, &min, &max);
        if (ret)
                return ret;

        skb2 = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!skb2)
                return -ENOMEM;

        nlh2 = start_msg(skb2, NETLINK_CB(skb).portid, info->nlh->nlmsg_seq, 0,
                         IPSET_CMD_TYPE);
        if (!nlh2)
                goto nlmsg_failure;
        if (nla_put_u8(skb2, IPSET_ATTR_PROTOCOL, protocol(attr)) ||
            nla_put_string(skb2, IPSET_ATTR_TYPENAME, typename) ||
            nla_put_u8(skb2, IPSET_ATTR_FAMILY, family) ||
            nla_put_u8(skb2, IPSET_ATTR_REVISION, max) ||
            nla_put_u8(skb2, IPSET_ATTR_REVISION_MIN, min))
                goto nla_put_failure;
        nlmsg_end(skb2, nlh2);

        pr_debug("Send TYPE, nlmsg_len: %u\n", nlh2->nlmsg_len);
        return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);

nla_put_failure:
        nlmsg_cancel(skb2, nlh2);
nlmsg_failure:
        kfree_skb(skb2);
        return -EMSGSIZE;
}

/* Get protocol version */

static const struct nla_policy
ip_set_protocol_policy[IPSET_ATTR_CMD_MAX + 1] = {
        [IPSET_ATTR_PROTOCOL]        = { .type = NLA_U8 },
};

static int ip_set_protocol(struct sk_buff *skb, const struct nfnl_info *info,
                           const struct nlattr * const attr[])
{
        struct sk_buff *skb2;
        struct nlmsghdr *nlh2;

        if (unlikely(!attr[IPSET_ATTR_PROTOCOL]))
                return -IPSET_ERR_PROTOCOL;

        skb2 = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!skb2)
                return -ENOMEM;

        nlh2 = start_msg(skb2, NETLINK_CB(skb).portid, info->nlh->nlmsg_seq, 0,
                         IPSET_CMD_PROTOCOL);
        if (!nlh2)
                goto nlmsg_failure;
        if (nla_put_u8(skb2, IPSET_ATTR_PROTOCOL, IPSET_PROTOCOL))
                goto nla_put_failure;
        if (nla_put_u8(skb2, IPSET_ATTR_PROTOCOL_MIN, IPSET_PROTOCOL_MIN))
                goto nla_put_failure;
        nlmsg_end(skb2, nlh2);

        return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);

nla_put_failure:
        nlmsg_cancel(skb2, nlh2);
nlmsg_failure:
        kfree_skb(skb2);
        return -EMSGSIZE;
}

/* Get set by name or index, from userspace */

static int ip_set_byname(struct sk_buff *skb, const struct nfnl_info *info,
                         const struct nlattr * const attr[])
{
        struct ip_set_net *inst = ip_set_pernet(info->net);
        struct sk_buff *skb2;
        struct nlmsghdr *nlh2;
        ip_set_id_t id = IPSET_INVALID_ID;
        const struct ip_set *set;

        if (unlikely(protocol_failed(attr) ||
                     !attr[IPSET_ATTR_SETNAME]))
                return -IPSET_ERR_PROTOCOL;

        set = find_set_and_id(inst, nla_data(attr[IPSET_ATTR_SETNAME]), &id);
        if (id == IPSET_INVALID_ID)
                return -ENOENT;

        skb2 = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!skb2)
                return -ENOMEM;

        nlh2 = start_msg(skb2, NETLINK_CB(skb).portid, info->nlh->nlmsg_seq, 0,
                         IPSET_CMD_GET_BYNAME);
        if (!nlh2)
                goto nlmsg_failure;
        if (nla_put_u8(skb2, IPSET_ATTR_PROTOCOL, protocol(attr)) ||
            nla_put_u8(skb2, IPSET_ATTR_FAMILY, set->family) ||
            nla_put_net16(skb2, IPSET_ATTR_INDEX, htons(id)))
                goto nla_put_failure;
        nlmsg_end(skb2, nlh2);

        return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);

nla_put_failure:
        nlmsg_cancel(skb2, nlh2);
nlmsg_failure:
        kfree_skb(skb2);
        return -EMSGSIZE;
}

static const struct nla_policy ip_set_index_policy[IPSET_ATTR_CMD_MAX + 1] = {
        [IPSET_ATTR_PROTOCOL]        = { .type = NLA_U8 },
        [IPSET_ATTR_INDEX]        = { .type = NLA_U16 },
};

static int ip_set_byindex(struct sk_buff *skb, const struct nfnl_info *info,
                          const struct nlattr * const attr[])
{
        struct ip_set_net *inst = ip_set_pernet(info->net);
        struct sk_buff *skb2;
        struct nlmsghdr *nlh2;
        ip_set_id_t id = IPSET_INVALID_ID;
        const struct ip_set *set;

        if (unlikely(protocol_failed(attr) ||
                     !attr[IPSET_ATTR_INDEX]))
                return -IPSET_ERR_PROTOCOL;

        id = ip_set_get_h16(attr[IPSET_ATTR_INDEX]);
        if (id >= inst->ip_set_max)
                return -ENOENT;
        set = ip_set(inst, id);
        if (set == NULL)
                return -ENOENT;

        skb2 = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!skb2)
                return -ENOMEM;

        nlh2 = start_msg(skb2, NETLINK_CB(skb).portid, info->nlh->nlmsg_seq, 0,
                         IPSET_CMD_GET_BYINDEX);
        if (!nlh2)
                goto nlmsg_failure;
        if (nla_put_u8(skb2, IPSET_ATTR_PROTOCOL, protocol(attr)) ||
            nla_put_string(skb2, IPSET_ATTR_SETNAME, set->name))
                goto nla_put_failure;
        nlmsg_end(skb2, nlh2);

        return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);

nla_put_failure:
        nlmsg_cancel(skb2, nlh2);
nlmsg_failure:
        kfree_skb(skb2);
        return -EMSGSIZE;
}

static const struct nfnl_callback ip_set_netlink_subsys_cb[IPSET_MSG_MAX] = {
        [IPSET_CMD_NONE]        = {
                .call                = ip_set_none,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = IPSET_ATTR_CMD_MAX,
        },
        [IPSET_CMD_CREATE]        = {
                .call                = ip_set_create,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = IPSET_ATTR_CMD_MAX,
                .policy                = ip_set_create_policy,
        },
        [IPSET_CMD_DESTROY]        = {
                .call                = ip_set_destroy,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = IPSET_ATTR_CMD_MAX,
                .policy                = ip_set_setname_policy,
        },
        [IPSET_CMD_FLUSH]        = {
                .call                = ip_set_flush,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = IPSET_ATTR_CMD_MAX,
                .policy                = ip_set_setname_policy,
        },
        [IPSET_CMD_RENAME]        = {
                .call                = ip_set_rename,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = IPSET_ATTR_CMD_MAX,
                .policy                = ip_set_setname2_policy,
        },
        [IPSET_CMD_SWAP]        = {
                .call                = ip_set_swap,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = IPSET_ATTR_CMD_MAX,
                .policy                = ip_set_setname2_policy,
        },
        [IPSET_CMD_LIST]        = {
                .call                = ip_set_dump,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = IPSET_ATTR_CMD_MAX,
                .policy                = ip_set_dump_policy,
        },
        [IPSET_CMD_SAVE]        = {
                .call                = ip_set_dump,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = IPSET_ATTR_CMD_MAX,
                .policy                = ip_set_setname_policy,
        },
        [IPSET_CMD_ADD]        = {
                .call                = ip_set_uadd,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = IPSET_ATTR_CMD_MAX,
                .policy                = ip_set_adt_policy,
        },
        [IPSET_CMD_DEL]        = {
                .call                = ip_set_udel,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = IPSET_ATTR_CMD_MAX,
                .policy                = ip_set_adt_policy,
        },
        [IPSET_CMD_TEST]        = {
                .call                = ip_set_utest,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = IPSET_ATTR_CMD_MAX,
                .policy                = ip_set_adt_policy,
        },
        [IPSET_CMD_HEADER]        = {
                .call                = ip_set_header,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = IPSET_ATTR_CMD_MAX,
                .policy                = ip_set_setname_policy,
        },
        [IPSET_CMD_TYPE]        = {
                .call                = ip_set_type,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = IPSET_ATTR_CMD_MAX,
                .policy                = ip_set_type_policy,
        },
        [IPSET_CMD_PROTOCOL]        = {
                .call                = ip_set_protocol,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = IPSET_ATTR_CMD_MAX,
                .policy                = ip_set_protocol_policy,
        },
        [IPSET_CMD_GET_BYNAME]        = {
                .call                = ip_set_byname,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = IPSET_ATTR_CMD_MAX,
                .policy                = ip_set_setname_policy,
        },
        [IPSET_CMD_GET_BYINDEX]        = {
                .call                = ip_set_byindex,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = IPSET_ATTR_CMD_MAX,
                .policy                = ip_set_index_policy,
        },
};

static struct nfnetlink_subsystem ip_set_netlink_subsys __read_mostly = {
        .name                = "ip_set",
        .subsys_id        = NFNL_SUBSYS_IPSET,
        .cb_count        = IPSET_MSG_MAX,
        .cb                = ip_set_netlink_subsys_cb,
};

/* Interface to iptables/ip6tables */

static int
ip_set_sockfn_get(struct sock *sk, int optval, void __user *user, int *len)
{
        unsigned int *op;
        void *data;
        int copylen = *len, ret = 0;
        struct net *net = sock_net(sk);
        struct ip_set_net *inst = ip_set_pernet(net);

        if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
                return -EPERM;
        if (optval != SO_IP_SET)
                return -EBADF;
        if (*len < sizeof(unsigned int))
                return -EINVAL;

        data = vmalloc(*len);
        if (!data)
                return -ENOMEM;
        if (copy_from_user(data, user, *len) != 0) {
                ret = -EFAULT;
                goto done;
        }
        op = data;

        if (*op < IP_SET_OP_VERSION) {
                /* Check the version at the beginning of operations */
                struct ip_set_req_version *req_version = data;

                if (*len < sizeof(struct ip_set_req_version)) {
                        ret = -EINVAL;
                        goto done;
                }

                if (req_version->version < IPSET_PROTOCOL_MIN) {
                        ret = -EPROTO;
                        goto done;
                }
        }

        switch (*op) {
        case IP_SET_OP_VERSION: {
                struct ip_set_req_version *req_version = data;

                if (*len != sizeof(struct ip_set_req_version)) {
                        ret = -EINVAL;
                        goto done;
                }

                req_version->version = IPSET_PROTOCOL;
                if (copy_to_user(user, req_version,
                                 sizeof(struct ip_set_req_version)))
                        ret = -EFAULT;
                goto done;
        }
        case IP_SET_OP_GET_BYNAME: {
                struct ip_set_req_get_set *req_get = data;
                ip_set_id_t id;

                if (*len != sizeof(struct ip_set_req_get_set)) {
                        ret = -EINVAL;
                        goto done;
                }
                req_get->set.name[IPSET_MAXNAMELEN - 1] = '\0';
                nfnl_lock(NFNL_SUBSYS_IPSET);
                find_set_and_id(inst, req_get->set.name, &id);
                req_get->set.index = id;
                nfnl_unlock(NFNL_SUBSYS_IPSET);
                goto copy;
        }
        case IP_SET_OP_GET_FNAME: {
                struct ip_set_req_get_set_family *req_get = data;
                ip_set_id_t id;

                if (*len != sizeof(struct ip_set_req_get_set_family)) {
                        ret = -EINVAL;
                        goto done;
                }
                req_get->set.name[IPSET_MAXNAMELEN - 1] = '\0';
                nfnl_lock(NFNL_SUBSYS_IPSET);
                find_set_and_id(inst, req_get->set.name, &id);
                req_get->set.index = id;
                if (id != IPSET_INVALID_ID)
                        req_get->family = ip_set(inst, id)->family;
                nfnl_unlock(NFNL_SUBSYS_IPSET);
                goto copy;
        }
        case IP_SET_OP_GET_BYINDEX: {
                struct ip_set_req_get_set *req_get = data;
                struct ip_set *set;

                if (*len != sizeof(struct ip_set_req_get_set) ||
                    req_get->set.index >= inst->ip_set_max) {
                        ret = -EINVAL;
                        goto done;
                }
                nfnl_lock(NFNL_SUBSYS_IPSET);
                set = ip_set(inst, req_get->set.index);
                ret = strscpy(req_get->set.name, set ? set->name : "",
                              IPSET_MAXNAMELEN);
                nfnl_unlock(NFNL_SUBSYS_IPSET);
                if (ret < 0)
                        goto done;
                goto copy;
        }
        default:
                ret = -EBADMSG;
                goto done;
        }        /* end of switch(op) */

copy:
        if (copy_to_user(user, data, copylen))
                ret = -EFAULT;

done:
        vfree(data);
        if (ret > 0)
                ret = 0;
        return ret;
}

static struct nf_sockopt_ops so_set __read_mostly = {
        .pf                = PF_INET,
        .get_optmin        = SO_IP_SET,
        .get_optmax        = SO_IP_SET + 1,
        .get                = ip_set_sockfn_get,
        .owner                = THIS_MODULE,
};

static int __net_init
ip_set_net_init(struct net *net)
{
        struct ip_set_net *inst = ip_set_pernet(net);
        struct ip_set **list;

        inst->ip_set_max = max_sets ? max_sets : CONFIG_IP_SET_MAX;
        if (inst->ip_set_max >= IPSET_INVALID_ID)
                inst->ip_set_max = IPSET_INVALID_ID - 1;

        list = kvcalloc(inst->ip_set_max, sizeof(struct ip_set *), GFP_KERNEL);
        if (!list)
                return -ENOMEM;
        inst->is_deleted = false;
        inst->is_destroyed = false;
        rcu_assign_pointer(inst->ip_set_list, list);
        return 0;
}

static void __net_exit
ip_set_net_pre_exit(struct net *net)
{
        struct ip_set_net *inst = ip_set_pernet(net);

        inst->is_deleted = true; /* flag for ip_set_nfnl_put */
}

static void __net_exit
ip_set_net_exit(struct net *net)
{
        struct ip_set_net *inst = ip_set_pernet(net);

        _destroy_all_sets(inst);
        kvfree(rcu_dereference_protected(inst->ip_set_list, 1));
}

static struct pernet_operations ip_set_net_ops = {
        .init        = ip_set_net_init,
        .pre_exit = ip_set_net_pre_exit,
        .exit   = ip_set_net_exit,
        .id        = &ip_set_net_id,
        .size        = sizeof(struct ip_set_net),
};

static int __init
ip_set_init(void)
{
        int ret = register_pernet_subsys(&ip_set_net_ops);

        if (ret) {
                pr_err("ip_set: cannot register pernet_subsys.\n");
                return ret;
        }

        ret = nfnetlink_subsys_register(&ip_set_netlink_subsys);
        if (ret != 0) {
                pr_err("ip_set: cannot register with nfnetlink.\n");
                unregister_pernet_subsys(&ip_set_net_ops);
                return ret;
        }

        ret = nf_register_sockopt(&so_set);
        if (ret != 0) {
                pr_err("SO_SET registry failed: %d\n", ret);
                nfnetlink_subsys_unregister(&ip_set_netlink_subsys);
                unregister_pernet_subsys(&ip_set_net_ops);
                return ret;
        }

        return 0;
}

static void __exit
ip_set_fini(void)
{
        nf_unregister_sockopt(&so_set);
        nfnetlink_subsys_unregister(&ip_set_netlink_subsys);
        unregister_pernet_subsys(&ip_set_net_ops);

        /* Wait for call_rcu() in destroy */
        rcu_barrier();

        pr_debug("these are the famous last words\n");
}

module_init(ip_set_init);
module_exit(ip_set_fini);

MODULE_DESCRIPTION("ip_set: protocol " __stringify(IPSET_PROTOCOL));


































  339 




















































 3520 


























 3521 































 1179 
 1181 


















































































   11 



 2593 

































 2601 
 2604 




















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  fs/anon_inodes.c
 *
 *  Copyright (C) 2007  Davide Libenzi <davidel@xmailserver.org>
 *
 *  Thanks to Arnd Bergmann for code review and suggestions.
 *  More changes for Thomas Gleixner suggestions.
 *
 */

#include <linux/cred.h>
#include <linux/file.h>
#include <linux/poll.h>
#include <linux/sched.h>
#include <linux/init.h>
#include <linux/fs.h>
#include <linux/mount.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/magic.h>
#include <linux/anon_inodes.h>
#include <linux/pseudo_fs.h>

#include <linux/uaccess.h>

static struct vfsmount *anon_inode_mnt __ro_after_init;
static struct inode *anon_inode_inode __ro_after_init;

/*
 * anon_inodefs_dname() is called from d_path().
 */
static char *anon_inodefs_dname(struct dentry *dentry, char *buffer, int buflen)
{
        return dynamic_dname(buffer, buflen, "anon_inode:%s",
                                dentry->d_name.name);
}

static const struct dentry_operations anon_inodefs_dentry_operations = {
        .d_dname        = anon_inodefs_dname,
};

static int anon_inodefs_init_fs_context(struct fs_context *fc)
{
        struct pseudo_fs_context *ctx = init_pseudo(fc, ANON_INODE_FS_MAGIC);
        if (!ctx)
                return -ENOMEM;
        ctx->dops = &anon_inodefs_dentry_operations;
        return 0;
}

static struct file_system_type anon_inode_fs_type = {
        .name                = "anon_inodefs",
        .init_fs_context = anon_inodefs_init_fs_context,
        .kill_sb        = kill_anon_super,
};

static struct inode *anon_inode_make_secure_inode(
        const char *name,
        const struct inode *context_inode)
{
        struct inode *inode;
        int error;

        inode = alloc_anon_inode(anon_inode_mnt->mnt_sb);
        if (IS_ERR(inode))
                return inode;
        inode->i_flags &= ~S_PRIVATE;
        error =        security_inode_init_security_anon(inode, &QSTR(name),
                                                  context_inode);
        if (error) {
                iput(inode);
                return ERR_PTR(error);
        }
        return inode;
}

static struct file *__anon_inode_getfile(const char *name,
                                         const struct file_operations *fops,
                                         void *priv, int flags,
                                         const struct inode *context_inode,
                                         bool make_inode)
{
        struct inode *inode;
        struct file *file;

        if (fops->owner && !try_module_get(fops->owner))
                return ERR_PTR(-ENOENT);

        if (make_inode) {
                inode =        anon_inode_make_secure_inode(name, context_inode);
                if (IS_ERR(inode)) {
                        file = ERR_CAST(inode);
                        goto err;
                }
        } else {
                inode =        anon_inode_inode;
                if (IS_ERR(inode)) {
                        file = ERR_PTR(-ENODEV);
                        goto err;
                }
                /*
                 * We know the anon_inode inode count is always
                 * greater than zero, so ihold() is safe.
                 */
                ihold(inode);
        }

        file = alloc_file_pseudo(inode, anon_inode_mnt, name,
                                 flags & (O_ACCMODE | O_NONBLOCK), fops);
        if (IS_ERR(file))
                goto err_iput;

        file->f_mapping = inode->i_mapping;

        file->private_data = priv;

        return file;

err_iput:
        iput(inode);
err:
        module_put(fops->owner);
        return file;
}

/**
 * anon_inode_getfile - creates a new file instance by hooking it up to an
 *                      anonymous inode, and a dentry that describe the "class"
 *                      of the file
 *
 * @name:    [in]    name of the "class" of the new file
 * @fops:    [in]    file operations for the new file
 * @priv:    [in]    private data for the new file (will be file's private_data)
 * @flags:   [in]    flags
 *
 * Creates a new file by hooking it on a single inode. This is useful for files
 * that do not need to have a full-fledged inode in order to operate correctly.
 * All the files created with anon_inode_getfile() will share a single inode,
 * hence saving memory and avoiding code duplication for the file/inode/dentry
 * setup.  Returns the newly created file* or an error pointer.
 */
struct file *anon_inode_getfile(const char *name,
                                const struct file_operations *fops,
                                void *priv, int flags)
{
        return __anon_inode_getfile(name, fops, priv, flags, NULL, false);
}
EXPORT_SYMBOL_GPL(anon_inode_getfile);

/**
 * anon_inode_getfile_fmode - creates a new file instance by hooking it up to an
 *                      anonymous inode, and a dentry that describe the "class"
 *                      of the file
 *
 * @name:    [in]    name of the "class" of the new file
 * @fops:    [in]    file operations for the new file
 * @priv:    [in]    private data for the new file (will be file's private_data)
 * @flags:   [in]    flags
 * @f_mode:  [in]    fmode
 *
 * Creates a new file by hooking it on a single inode. This is useful for files
 * that do not need to have a full-fledged inode in order to operate correctly.
 * All the files created with anon_inode_getfile() will share a single inode,
 * hence saving memory and avoiding code duplication for the file/inode/dentry
 * setup. Allows setting the fmode. Returns the newly created file* or an error
 * pointer.
 */
struct file *anon_inode_getfile_fmode(const char *name,
                                const struct file_operations *fops,
                                void *priv, int flags, fmode_t f_mode)
{
        struct file *file;

        file = __anon_inode_getfile(name, fops, priv, flags, NULL, false);
        if (!IS_ERR(file))
                file->f_mode |= f_mode;

        return file;
}
EXPORT_SYMBOL_GPL(anon_inode_getfile_fmode);

/**
 * anon_inode_create_getfile - Like anon_inode_getfile(), but creates a new
 *                             !S_PRIVATE anon inode rather than reuse the
 *                             singleton anon inode and calls the
 *                             inode_init_security_anon() LSM hook.
 *
 * @name:    [in]    name of the "class" of the new file
 * @fops:    [in]    file operations for the new file
 * @priv:    [in]    private data for the new file (will be file's private_data)
 * @flags:   [in]    flags
 * @context_inode:
 *           [in]    the logical relationship with the new inode (optional)
 *
 * Create a new anonymous inode and file pair.  This can be done for two
 * reasons:
 *
 * - for the inode to have its own security context, so that LSMs can enforce
 *   policy on the inode's creation;
 *
 * - if the caller needs a unique inode, for example in order to customize
 *   the size returned by fstat()
 *
 * The LSM may use @context_inode in inode_init_security_anon(), but a
 * reference to it is not held.
 *
 * Returns the newly created file* or an error pointer.
 */
struct file *anon_inode_create_getfile(const char *name,
                                       const struct file_operations *fops,
                                       void *priv, int flags,
                                       const struct inode *context_inode)
{
        return __anon_inode_getfile(name, fops, priv, flags,
                                    context_inode, true);
}
EXPORT_SYMBOL_GPL(anon_inode_create_getfile);

static int __anon_inode_getfd(const char *name,
                              const struct file_operations *fops,
                              void *priv, int flags,
                              const struct inode *context_inode,
                              bool make_inode)
{
        int error, fd;
        struct file *file;

        error = get_unused_fd_flags(flags);
        if (error < 0)
                return error;
        fd = error;

        file = __anon_inode_getfile(name, fops, priv, flags, context_inode,
                                    make_inode);
        if (IS_ERR(file)) {
                error = PTR_ERR(file);
                goto err_put_unused_fd;
        }
        fd_install(fd, file);

        return fd;

err_put_unused_fd:
        put_unused_fd(fd);
        return error;
}

/**
 * anon_inode_getfd - creates a new file instance by hooking it up to
 *                    an anonymous inode and a dentry that describe
 *                    the "class" of the file
 *
 * @name:    [in]    name of the "class" of the new file
 * @fops:    [in]    file operations for the new file
 * @priv:    [in]    private data for the new file (will be file's private_data)
 * @flags:   [in]    flags
 *
 * Creates a new file by hooking it on a single inode. This is
 * useful for files that do not need to have a full-fledged inode in
 * order to operate correctly.  All the files created with
 * anon_inode_getfd() will use the same singleton inode, reducing
 * memory use and avoiding code duplication for the file/inode/dentry
 * setup.  Returns a newly created file descriptor or an error code.
 */
int anon_inode_getfd(const char *name, const struct file_operations *fops,
                     void *priv, int flags)
{
        return __anon_inode_getfd(name, fops, priv, flags, NULL, false);
}
EXPORT_SYMBOL_GPL(anon_inode_getfd);

/**
 * anon_inode_create_getfd - Like anon_inode_getfd(), but creates a new
 * !S_PRIVATE anon inode rather than reuse the singleton anon inode, and calls
 * the inode_init_security_anon() LSM hook.
 *
 * @name:    [in]    name of the "class" of the new file
 * @fops:    [in]    file operations for the new file
 * @priv:    [in]    private data for the new file (will be file's private_data)
 * @flags:   [in]    flags
 * @context_inode:
 *           [in]    the logical relationship with the new inode (optional)
 *
 * Create a new anonymous inode and file pair.  This can be done for two
 * reasons:
 *
 * - for the inode to have its own security context, so that LSMs can enforce
 *   policy on the inode's creation;
 *
 * - if the caller needs a unique inode, for example in order to customize
 *   the size returned by fstat()
 *
 * The LSM may use @context_inode in inode_init_security_anon(), but a
 * reference to it is not held.
 *
 * Returns a newly created file descriptor or an error code.
 */
int anon_inode_create_getfd(const char *name, const struct file_operations *fops,
                            void *priv, int flags,
                            const struct inode *context_inode)
{
        return __anon_inode_getfd(name, fops, priv, flags, context_inode, true);
}


static int __init anon_inode_init(void)
{
        anon_inode_mnt = kern_mount(&anon_inode_fs_type);
        if (IS_ERR(anon_inode_mnt))
                panic("anon_inode_init() kernel mount failed (%ld)\n", PTR_ERR(anon_inode_mnt));

        anon_inode_inode = alloc_anon_inode(anon_inode_mnt->mnt_sb);
        if (IS_ERR(anon_inode_inode))
                panic("anon_inode_init() inode allocation failed (%ld)\n", PTR_ERR(anon_inode_inode));

        return 0;
}

fs_initcall(anon_inode_init);


































































































































































































































   88 




   74 










































   21 













   18 




    2 




   13 




    6 




























   25 













   18 





   11 



























   25 





   60 







































   38 






















































   61 









































































   21 




















   21 



































































































































































   10 






















    6 





















































































































   37 





















































































































   41 













   37 






   38 




















   29 


































































































































    4 





















   46 





































































































































































    1 






















   12 






























































































































































































































































   38 



































































































































































   38 


























































































    2 





































   38 





















































































































































































































































































































































































   24 























































































































































































































































































































































































































































































































































































































































































































































































































































































   50 


























   50 








































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Portions of this file
 * Copyright(c) 2016-2017 Intel Deutschland GmbH
 * Copyright (C) 2018 - 2024 Intel Corporation
 */

#if !defined(__MAC80211_DRIVER_TRACE) || defined(TRACE_HEADER_MULTI_READ)
#define __MAC80211_DRIVER_TRACE

#include <linux/tracepoint.h>
#include <net/mac80211.h>
#include "ieee80211_i.h"

#undef TRACE_SYSTEM
#define TRACE_SYSTEM mac80211

#define MAXNAME                32
#define LOCAL_ENTRY        __array(char, wiphy_name, 32)
#define LOCAL_ASSIGN        strscpy(__entry->wiphy_name, wiphy_name(local->hw.wiphy), MAXNAME)
#define LOCAL_PR_FMT        "%s"
#define LOCAL_PR_ARG        __entry->wiphy_name

#define STA_ENTRY        __array(char, sta_addr, ETH_ALEN)
#define STA_ASSIGN        (sta ? memcpy(__entry->sta_addr, sta->addr, ETH_ALEN) : \
                                eth_zero_addr(__entry->sta_addr))
#define STA_NAMED_ASSIGN(s)        memcpy(__entry->sta_addr, (s)->addr, ETH_ALEN)
#define STA_PR_FMT        " sta:%pM"
#define STA_PR_ARG        __entry->sta_addr

#define VIF_ENTRY        __field(enum nl80211_iftype, vif_type) __field(void *, sdata)        \
                        __field(bool, p2p)                                                \
                        __string(vif_name, sdata->name)
#define VIF_ASSIGN        __entry->vif_type = sdata->vif.type; __entry->sdata = sdata;        \
                        __entry->p2p = sdata->vif.p2p;                                        \
                        __assign_str(vif_name)
#define VIF_PR_FMT        " vif:%s(%d%s)"
#define VIF_PR_ARG        __get_str(vif_name), __entry->vif_type, __entry->p2p ? "/p2p" : ""

#define CHANDEF_ENTRY        __field(u32, control_freq)                                        \
                        __field(u32, freq_offset)                                        \
                        __field(u32, chan_width)                                        \
                        __field(u32, center_freq1)                                        \
                        __field(u32, freq1_offset)                                        \
                        __field(u32, center_freq2)
#define CHANDEF_ASSIGN(c)                                                        \
                        __entry->control_freq = (c) ? ((c)->chan ? (c)->chan->center_freq : 0) : 0;        \
                        __entry->freq_offset = (c) ? ((c)->chan ? (c)->chan->freq_offset : 0) : 0;        \
                        __entry->chan_width = (c) ? (c)->width : 0;                        \
                        __entry->center_freq1 = (c) ? (c)->center_freq1 : 0;                \
                        __entry->freq1_offset = (c) ? (c)->freq1_offset : 0;                \
                        __entry->center_freq2 = (c) ? (c)->center_freq2 : 0;
#define CHANDEF_PR_FMT        " chandef(%d.%03d MHz,width:%d,center: %d.%03d/%d MHz)"
#define CHANDEF_PR_ARG        __entry->control_freq, __entry->freq_offset, __entry->chan_width, \
                        __entry->center_freq1, __entry->freq1_offset, __entry->center_freq2

#define MIN_CHANDEF_ENTRY                                                                \
                        __field(u32, min_control_freq)                                        \
                        __field(u32, min_freq_offset)                                        \
                        __field(u32, min_chan_width)                                        \
                        __field(u32, min_center_freq1)                                        \
                        __field(u32, min_freq1_offset)                                        \
                        __field(u32, min_center_freq2)

#define MIN_CHANDEF_ASSIGN(c)                                                                \
                        __entry->min_control_freq = (c)->chan ? (c)->chan->center_freq : 0;        \
                        __entry->min_freq_offset = (c)->chan ? (c)->chan->freq_offset : 0;        \
                        __entry->min_chan_width = (c)->width;                                \
                        __entry->min_center_freq1 = (c)->center_freq1;                        \
                        __entry->min_freq1_offset = (c)->freq1_offset;                        \
                        __entry->min_center_freq2 = (c)->center_freq2;
#define MIN_CHANDEF_PR_FMT        " mindef(%d.%03d MHz,width:%d,center: %d.%03d/%d MHz)"
#define MIN_CHANDEF_PR_ARG        __entry->min_control_freq, __entry->min_freq_offset,        \
                        __entry->min_chan_width,                                        \
                        __entry->min_center_freq1, __entry->min_freq1_offset,                \
                        __entry->min_center_freq2

#define AP_CHANDEF_ENTRY                                                                \
                        __field(u32, ap_control_freq)                                        \
                        __field(u32, ap_freq_offset)                                        \
                        __field(u32, ap_chan_width)                                        \
                        __field(u32, ap_center_freq1)                                        \
                        __field(u32, ap_freq1_offset)                                        \
                        __field(u32, ap_center_freq2)

#define AP_CHANDEF_ASSIGN(c)                                                                \
                        __entry->ap_control_freq = (c)->chan ? (c)->chan->center_freq : 0;\
                        __entry->ap_freq_offset = (c)->chan ? (c)->chan->freq_offset : 0;\
                        __entry->ap_chan_width = (c)->chan ? (c)->width : 0;                \
                        __entry->ap_center_freq1 = (c)->chan ? (c)->center_freq1 : 0;        \
                        __entry->ap_freq1_offset = (c)->chan ? (c)->freq1_offset : 0;        \
                        __entry->ap_center_freq2 = (c)->chan ? (c)->center_freq2 : 0;
#define AP_CHANDEF_PR_FMT        " ap(%d.%03d MHz,width:%d,center: %d.%03d/%d MHz)"
#define AP_CHANDEF_PR_ARG        __entry->ap_control_freq, __entry->ap_freq_offset,        \
                        __entry->ap_chan_width,                                                \
                        __entry->ap_center_freq1, __entry->ap_freq1_offset,                \
                        __entry->ap_center_freq2

#define CHANCTX_ENTRY        CHANDEF_ENTRY                                                        \
                        MIN_CHANDEF_ENTRY                                                \
                        AP_CHANDEF_ENTRY                                                \
                        __field(u8, rx_chains_static)                                        \
                        __field(u8, rx_chains_dynamic)
#define CHANCTX_ASSIGN        CHANDEF_ASSIGN(&ctx->conf.def)                                        \
                        MIN_CHANDEF_ASSIGN(&ctx->conf.min_def)                                \
                        AP_CHANDEF_ASSIGN(&ctx->conf.ap)                                \
                        __entry->rx_chains_static = ctx->conf.rx_chains_static;                \
                        __entry->rx_chains_dynamic = ctx->conf.rx_chains_dynamic
#define CHANCTX_PR_FMT        CHANDEF_PR_FMT MIN_CHANDEF_PR_FMT AP_CHANDEF_PR_FMT " chains:%d/%d"
#define CHANCTX_PR_ARG        CHANDEF_PR_ARG,        MIN_CHANDEF_PR_ARG, AP_CHANDEF_PR_ARG,                \
                        __entry->rx_chains_static, __entry->rx_chains_dynamic

#define KEY_ENTRY        __field(u32, cipher)                                                \
                        __field(u8, hw_key_idx)                                                \
                        __field(u8, flags)                                                \
                        __field(s8, keyidx)
#define KEY_ASSIGN(k)        __entry->cipher = (k)->cipher;                                        \
                        __entry->flags = (k)->flags;                                        \
                        __entry->keyidx = (k)->keyidx;                                        \
                        __entry->hw_key_idx = (k)->hw_key_idx;
#define KEY_PR_FMT        " cipher:0x%x, flags=%#x, keyidx=%d, hw_key_idx=%d"
#define KEY_PR_ARG        __entry->cipher, __entry->flags, __entry->keyidx, __entry->hw_key_idx

#define AMPDU_ACTION_ENTRY        __field(enum ieee80211_ampdu_mlme_action,                \
                                        ieee80211_ampdu_mlme_action)                        \
                                STA_ENTRY                                                \
                                __field(u16, tid)                                        \
                                __field(u16, ssn)                                        \
                                __field(u16, buf_size)                                        \
                                __field(bool, amsdu)                                        \
                                __field(u16, timeout)                                        \
                                __field(u16, action)
#define AMPDU_ACTION_ASSIGN        STA_NAMED_ASSIGN(params->sta);                                \
                                __entry->tid = params->tid;                                \
                                __entry->ssn = params->ssn;                                \
                                __entry->buf_size = params->buf_size;                        \
                                __entry->amsdu = params->amsdu;                                \
                                __entry->timeout = params->timeout;                        \
                                __entry->action = params->action;
#define AMPDU_ACTION_PR_FMT        STA_PR_FMT " tid %d, ssn %d, buf_size %u, amsdu %d, timeout %d action %d"
#define AMPDU_ACTION_PR_ARG        STA_PR_ARG, __entry->tid, __entry->ssn,                        \
                                __entry->buf_size, __entry->amsdu, __entry->timeout,        \
                                __entry->action

/*
 * Tracing for driver callbacks.
 */

DECLARE_EVENT_CLASS(local_only_evt,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local),
        TP_STRUCT__entry(
                LOCAL_ENTRY
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
        ),
        TP_printk(LOCAL_PR_FMT, LOCAL_PR_ARG)
);

DECLARE_EVENT_CLASS(local_sdata_addr_evt,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __array(char, addr, ETH_ALEN)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                memcpy(__entry->addr, sdata->vif.addr, ETH_ALEN);
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT " addr:%pM",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->addr
        )
);

DECLARE_EVENT_CLASS(local_u32_evt,
        TP_PROTO(struct ieee80211_local *local, u32 value),
        TP_ARGS(local, value),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u32, value)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->value = value;
        ),

        TP_printk(
                LOCAL_PR_FMT " value:%d",
                LOCAL_PR_ARG, __entry->value
        )
);

DECLARE_EVENT_CLASS(local_sdata_evt,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG
        )
);

DEFINE_EVENT(local_only_evt, drv_return_void,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

TRACE_EVENT(drv_return_int,
        TP_PROTO(struct ieee80211_local *local, int ret),
        TP_ARGS(local, ret),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(int, ret)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->ret = ret;
        ),
        TP_printk(LOCAL_PR_FMT " - %d", LOCAL_PR_ARG, __entry->ret)
);

TRACE_EVENT(drv_return_bool,
        TP_PROTO(struct ieee80211_local *local, bool ret),
        TP_ARGS(local, ret),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(bool, ret)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->ret = ret;
        ),
        TP_printk(LOCAL_PR_FMT " - %s", LOCAL_PR_ARG, (__entry->ret) ?
                  "true" : "false")
);

TRACE_EVENT(drv_return_u32,
        TP_PROTO(struct ieee80211_local *local, u32 ret),
        TP_ARGS(local, ret),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u32, ret)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->ret = ret;
        ),
        TP_printk(LOCAL_PR_FMT " - %u", LOCAL_PR_ARG, __entry->ret)
);

TRACE_EVENT(drv_return_u64,
        TP_PROTO(struct ieee80211_local *local, u64 ret),
        TP_ARGS(local, ret),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u64, ret)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->ret = ret;
        ),
        TP_printk(LOCAL_PR_FMT " - %llu", LOCAL_PR_ARG, __entry->ret)
);

DEFINE_EVENT(local_only_evt, drv_start,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

DEFINE_EVENT(local_u32_evt, drv_get_et_strings,
             TP_PROTO(struct ieee80211_local *local, u32 sset),
             TP_ARGS(local, sset)
);

DEFINE_EVENT(local_u32_evt, drv_get_et_sset_count,
             TP_PROTO(struct ieee80211_local *local, u32 sset),
             TP_ARGS(local, sset)
);

DEFINE_EVENT(local_only_evt, drv_get_et_stats,
             TP_PROTO(struct ieee80211_local *local),
             TP_ARGS(local)
);

DEFINE_EVENT(local_only_evt, drv_suspend,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

DEFINE_EVENT(local_only_evt, drv_resume,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

TRACE_EVENT(drv_set_wakeup,
        TP_PROTO(struct ieee80211_local *local, bool enabled),
        TP_ARGS(local, enabled),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(bool, enabled)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->enabled = enabled;
        ),
        TP_printk(LOCAL_PR_FMT " enabled:%d", LOCAL_PR_ARG, __entry->enabled)
);

TRACE_EVENT(drv_stop,
        TP_PROTO(struct ieee80211_local *local, bool suspend),
        TP_ARGS(local, suspend),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(bool, suspend)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->suspend = suspend;
        ),
        TP_printk(LOCAL_PR_FMT " suspend:%d", LOCAL_PR_ARG, __entry->suspend)
);

DEFINE_EVENT(local_sdata_addr_evt, drv_add_interface,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

TRACE_EVENT(drv_change_interface,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 enum nl80211_iftype type, bool p2p),

        TP_ARGS(local, sdata, type, p2p),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u32, new_type)
                __field(bool, new_p2p)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->new_type = type;
                __entry->new_p2p = p2p;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT " new type:%d%s",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->new_type,
                __entry->new_p2p ? "/p2p" : ""
        )
);

DEFINE_EVENT(local_sdata_addr_evt, drv_remove_interface,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

TRACE_EVENT(drv_config,
        TP_PROTO(struct ieee80211_local *local,
                 u32 changed),

        TP_ARGS(local, changed),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u32, changed)
                __field(u32, flags)
                __field(int, power_level)
                __field(int, dynamic_ps_timeout)
                __field(u16, listen_interval)
                __field(u8, long_frame_max_tx_count)
                __field(u8, short_frame_max_tx_count)
                CHANDEF_ENTRY
                __field(int, smps)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->changed = changed;
                __entry->flags = local->hw.conf.flags;
                __entry->power_level = local->hw.conf.power_level;
                __entry->dynamic_ps_timeout = local->hw.conf.dynamic_ps_timeout;
                __entry->listen_interval = local->hw.conf.listen_interval;
                __entry->long_frame_max_tx_count =
                        local->hw.conf.long_frame_max_tx_count;
                __entry->short_frame_max_tx_count =
                        local->hw.conf.short_frame_max_tx_count;
                CHANDEF_ASSIGN(&local->hw.conf.chandef)
                __entry->smps = local->hw.conf.smps_mode;
        ),

        TP_printk(
                LOCAL_PR_FMT " ch:%#x" CHANDEF_PR_FMT,
                LOCAL_PR_ARG, __entry->changed, CHANDEF_PR_ARG
        )
);

TRACE_EVENT(drv_vif_cfg_changed,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 u64 changed),

        TP_ARGS(local, sdata, changed),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u64, changed)
                __field(bool, assoc)
                __field(bool, ibss_joined)
                __field(bool, ibss_creator)
                __field(u16, aid)
                __dynamic_array(u32, arp_addr_list,
                                sdata->vif.cfg.arp_addr_cnt > IEEE80211_BSS_ARP_ADDR_LIST_LEN ?
                                        IEEE80211_BSS_ARP_ADDR_LIST_LEN :
                                        sdata->vif.cfg.arp_addr_cnt)
                __field(int, arp_addr_cnt)
                __dynamic_array(u8, ssid, sdata->vif.cfg.ssid_len)
                __field(int, s1g)
                __field(bool, idle)
                __field(bool, ps)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->changed = changed;
                __entry->aid = sdata->vif.cfg.aid;
                __entry->assoc = sdata->vif.cfg.assoc;
                __entry->ibss_joined = sdata->vif.cfg.ibss_joined;
                __entry->ibss_creator = sdata->vif.cfg.ibss_creator;
                __entry->ps = sdata->vif.cfg.ps;

                __entry->arp_addr_cnt = sdata->vif.cfg.arp_addr_cnt;
                memcpy(__get_dynamic_array(arp_addr_list),
                       sdata->vif.cfg.arp_addr_list,
                       sizeof(u32) * (sdata->vif.cfg.arp_addr_cnt > IEEE80211_BSS_ARP_ADDR_LIST_LEN ?
                                        IEEE80211_BSS_ARP_ADDR_LIST_LEN :
                                        sdata->vif.cfg.arp_addr_cnt));
                memcpy(__get_dynamic_array(ssid),
                       sdata->vif.cfg.ssid,
                       sdata->vif.cfg.ssid_len);
                __entry->s1g = sdata->vif.cfg.s1g;
                __entry->idle = sdata->vif.cfg.idle;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT " changed:%#llx",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->changed
        )
);

TRACE_EVENT(drv_link_info_changed,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_bss_conf *link_conf,
                 u64 changed),

        TP_ARGS(local, sdata, link_conf, changed),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u64, changed)
                __field(int, link_id)
                __field(bool, cts)
                __field(bool, shortpre)
                __field(bool, shortslot)
                __field(bool, enable_beacon)
                __field(u8, dtimper)
                __field(u16, bcnint)
                __field(u16, assoc_cap)
                __field(u64, sync_tsf)
                __field(u32, sync_device_ts)
                __field(u8, sync_dtim_count)
                __field(u32, basic_rates)
                __array(int, mcast_rate, NUM_NL80211_BANDS)
                __field(u16, ht_operation_mode)
                __field(s32, cqm_rssi_thold)
                __field(s32, cqm_rssi_hyst)
                __field(u32, channel_width)
                __field(u32, channel_cfreq1)
                __field(u32, channel_cfreq1_offset)
                __field(bool, qos)
                __field(bool, hidden_ssid)
                __field(int, txpower)
                __field(u8, p2p_oppps_ctwindow)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->changed = changed;
                __entry->link_id = link_conf->link_id;
                __entry->shortpre = link_conf->use_short_preamble;
                __entry->cts = link_conf->use_cts_prot;
                __entry->shortslot = link_conf->use_short_slot;
                __entry->enable_beacon = link_conf->enable_beacon;
                __entry->dtimper = link_conf->dtim_period;
                __entry->bcnint = link_conf->beacon_int;
                __entry->assoc_cap = link_conf->assoc_capability;
                __entry->sync_tsf = link_conf->sync_tsf;
                __entry->sync_device_ts = link_conf->sync_device_ts;
                __entry->sync_dtim_count = link_conf->sync_dtim_count;
                __entry->basic_rates = link_conf->basic_rates;
                memcpy(__entry->mcast_rate, link_conf->mcast_rate,
                       sizeof(__entry->mcast_rate));
                __entry->ht_operation_mode = link_conf->ht_operation_mode;
                __entry->cqm_rssi_thold = link_conf->cqm_rssi_thold;
                __entry->cqm_rssi_hyst = link_conf->cqm_rssi_hyst;
                __entry->channel_width = link_conf->chanreq.oper.width;
                __entry->channel_cfreq1 = link_conf->chanreq.oper.center_freq1;
                __entry->channel_cfreq1_offset = link_conf->chanreq.oper.freq1_offset;
                __entry->qos = link_conf->qos;
                __entry->hidden_ssid = link_conf->hidden_ssid;
                __entry->txpower = link_conf->txpower;
                __entry->p2p_oppps_ctwindow = link_conf->p2p_noa_attr.oppps_ctwindow;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT " link_id:%d, changed:%#llx",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->link_id,
                __entry->changed
        )
);

TRACE_EVENT(drv_prepare_multicast,
        TP_PROTO(struct ieee80211_local *local, int mc_count),

        TP_ARGS(local, mc_count),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(int, mc_count)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->mc_count = mc_count;
        ),

        TP_printk(
                LOCAL_PR_FMT " prepare mc (%d)",
                LOCAL_PR_ARG, __entry->mc_count
        )
);

TRACE_EVENT(drv_configure_filter,
        TP_PROTO(struct ieee80211_local *local,
                 unsigned int changed_flags,
                 unsigned int *total_flags,
                 u64 multicast),

        TP_ARGS(local, changed_flags, total_flags, multicast),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(unsigned int, changed)
                __field(unsigned int, total)
                __field(u64, multicast)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->changed = changed_flags;
                __entry->total = *total_flags;
                __entry->multicast = multicast;
        ),

        TP_printk(
                LOCAL_PR_FMT " changed:%#x total:%#x",
                LOCAL_PR_ARG, __entry->changed, __entry->total
        )
);

TRACE_EVENT(drv_config_iface_filter,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 unsigned int filter_flags,
                 unsigned int changed_flags),

        TP_ARGS(local, sdata, filter_flags, changed_flags),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(unsigned int, filter_flags)
                __field(unsigned int, changed_flags)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->filter_flags = filter_flags;
                __entry->changed_flags = changed_flags;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT
                " filter_flags: %#x changed_flags: %#x",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->filter_flags,
                __entry->changed_flags
        )
);

TRACE_EVENT(drv_set_tim,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta, bool set),

        TP_ARGS(local, sta, set),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                STA_ENTRY
                __field(bool, set)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                STA_ASSIGN;
                __entry->set = set;
        ),

        TP_printk(
                LOCAL_PR_FMT STA_PR_FMT " set:%d",
                LOCAL_PR_ARG, STA_PR_ARG, __entry->set
        )
);

TRACE_EVENT(drv_set_key,
        TP_PROTO(struct ieee80211_local *local,
                 enum set_key_cmd cmd, struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta,
                 struct ieee80211_key_conf *key),

        TP_ARGS(local, cmd, sdata, sta, key),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(u32, cmd)
                KEY_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
                __entry->cmd = cmd;
                KEY_ASSIGN(key);
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  STA_PR_FMT " cmd: %d" KEY_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG, __entry->cmd, KEY_PR_ARG
        )
);

TRACE_EVENT(drv_update_tkip_key,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_key_conf *conf,
                 struct ieee80211_sta *sta, u32 iv32),

        TP_ARGS(local, sdata, conf, sta, iv32),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(u32, iv32)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
                __entry->iv32 = iv32;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT STA_PR_FMT " iv32:%#x",
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG, __entry->iv32
        )
);

DEFINE_EVENT(local_sdata_evt, drv_hw_scan,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

DEFINE_EVENT(local_sdata_evt, drv_cancel_hw_scan,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

DEFINE_EVENT(local_sdata_evt, drv_sched_scan_start,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

DEFINE_EVENT(local_sdata_evt, drv_sched_scan_stop,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

TRACE_EVENT(drv_sw_scan_start,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 const u8 *mac_addr),

        TP_ARGS(local, sdata, mac_addr),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __array(char, mac_addr, ETH_ALEN)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                memcpy(__entry->mac_addr, mac_addr, ETH_ALEN);
        ),

        TP_printk(LOCAL_PR_FMT ", " VIF_PR_FMT ", addr:%pM",
                  LOCAL_PR_ARG, VIF_PR_ARG, __entry->mac_addr)
);

DEFINE_EVENT(local_sdata_evt, drv_sw_scan_complete,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

TRACE_EVENT(drv_get_stats,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_low_level_stats *stats,
                 int ret),

        TP_ARGS(local, stats, ret),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(int, ret)
                __field(unsigned int, ackfail)
                __field(unsigned int, rtsfail)
                __field(unsigned int, fcserr)
                __field(unsigned int, rtssucc)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->ret = ret;
                __entry->ackfail = stats->dot11ACKFailureCount;
                __entry->rtsfail = stats->dot11RTSFailureCount;
                __entry->fcserr = stats->dot11FCSErrorCount;
                __entry->rtssucc = stats->dot11RTSSuccessCount;
        ),

        TP_printk(
                LOCAL_PR_FMT " ret:%d",
                LOCAL_PR_ARG, __entry->ret
        )
);

TRACE_EVENT(drv_get_key_seq,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_key_conf *key),

        TP_ARGS(local, key),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                KEY_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                KEY_ASSIGN(key);
        ),

        TP_printk(
                LOCAL_PR_FMT KEY_PR_FMT,
                LOCAL_PR_ARG, KEY_PR_ARG
        )
);

DEFINE_EVENT(local_u32_evt, drv_set_frag_threshold,
        TP_PROTO(struct ieee80211_local *local, u32 value),
        TP_ARGS(local, value)
);

DEFINE_EVENT(local_u32_evt, drv_set_rts_threshold,
        TP_PROTO(struct ieee80211_local *local, u32 value),
        TP_ARGS(local, value)
);

TRACE_EVENT(drv_set_coverage_class,
        TP_PROTO(struct ieee80211_local *local, s16 value),

        TP_ARGS(local, value),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(s16, value)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->value = value;
        ),

        TP_printk(
                LOCAL_PR_FMT " value:%d",
                LOCAL_PR_ARG, __entry->value
        )
);

TRACE_EVENT(drv_sta_notify,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 enum sta_notify_cmd cmd,
                 struct ieee80211_sta *sta),

        TP_ARGS(local, sdata, cmd, sta),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(u32, cmd)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
                __entry->cmd = cmd;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  STA_PR_FMT " cmd:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG, __entry->cmd
        )
);

TRACE_EVENT(drv_sta_state,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta,
                 enum ieee80211_sta_state old_state,
                 enum ieee80211_sta_state new_state),

        TP_ARGS(local, sdata, sta, old_state, new_state),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(u32, old_state)
                __field(u32, new_state)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
                __entry->old_state = old_state;
                __entry->new_state = new_state;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  STA_PR_FMT " state: %d->%d",
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG,
                __entry->old_state, __entry->new_state
        )
);

TRACE_EVENT(drv_sta_set_txpwr,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),

        TP_ARGS(local, sdata, sta),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(s16, txpwr)
                __field(u8, type)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
                __entry->txpwr = sta->deflink.txpwr.power;
                __entry->type = sta->deflink.txpwr.type;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  STA_PR_FMT " txpwr: %d type %d",
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG,
                __entry->txpwr,  __entry->type
        )
);

TRACE_EVENT(drv_link_sta_rc_update,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_link_sta *link_sta,
                 u32 changed),

        TP_ARGS(local, sdata, link_sta, changed),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(u32, changed)
                __field(u32, link_id)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_NAMED_ASSIGN(link_sta->sta);
                __entry->changed = changed;
                __entry->link_id = link_sta->link_id;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  STA_PR_FMT " (link %d) changed: 0x%x",
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG, __entry->link_id,
                __entry->changed
        )
);

DECLARE_EVENT_CLASS(sta_event,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),

        TP_ARGS(local, sdata, sta),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  STA_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG
        )
);

DEFINE_EVENT(sta_event, drv_sta_statistics,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),
        TP_ARGS(local, sdata, sta)
);

DEFINE_EVENT(sta_event, drv_sta_add,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),
        TP_ARGS(local, sdata, sta)
);

DEFINE_EVENT(sta_event, drv_sta_remove,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),
        TP_ARGS(local, sdata, sta)
);

DEFINE_EVENT(sta_event, drv_sta_pre_rcu_remove,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),
        TP_ARGS(local, sdata, sta)
);

DEFINE_EVENT(sta_event, drv_sync_rx_queues,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),
        TP_ARGS(local, sdata, sta)
);

DEFINE_EVENT(sta_event, drv_sta_rate_tbl_update,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),
        TP_ARGS(local, sdata, sta)
);

TRACE_EVENT(drv_conf_tx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 unsigned int link_id,
                 u16 ac, const struct ieee80211_tx_queue_params *params),

        TP_ARGS(local, sdata, link_id, ac, params),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(unsigned int, link_id)
                __field(u16, ac)
                __field(u16, txop)
                __field(u16, cw_min)
                __field(u16, cw_max)
                __field(u8, aifs)
                __field(bool, uapsd)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->link_id = link_id;
                __entry->ac = ac;
                __entry->txop = params->txop;
                __entry->cw_max = params->cw_max;
                __entry->cw_min = params->cw_min;
                __entry->aifs = params->aifs;
                __entry->uapsd = params->uapsd;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  " link_id: %d, AC:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->link_id, __entry->ac
        )
);

DEFINE_EVENT(local_sdata_evt, drv_get_tsf,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

TRACE_EVENT(drv_set_tsf,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 u64 tsf),

        TP_ARGS(local, sdata, tsf),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u64, tsf)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->tsf = tsf;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  " tsf:%llu",
                LOCAL_PR_ARG, VIF_PR_ARG, (unsigned long long)__entry->tsf
        )
);

TRACE_EVENT(drv_offset_tsf,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 s64 offset),

        TP_ARGS(local, sdata, offset),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(s64, tsf_offset)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->tsf_offset = offset;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  " tsf offset:%lld",
                LOCAL_PR_ARG, VIF_PR_ARG,
                (unsigned long long)__entry->tsf_offset
        )
);

DEFINE_EVENT(local_sdata_evt, drv_reset_tsf,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

DEFINE_EVENT(local_only_evt, drv_tx_last_beacon,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

TRACE_EVENT(drv_ampdu_action,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_ampdu_params *params),

        TP_ARGS(local, sdata, params),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                AMPDU_ACTION_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                AMPDU_ACTION_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT AMPDU_ACTION_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG, AMPDU_ACTION_PR_ARG
        )
);

TRACE_EVENT(drv_get_survey,
        TP_PROTO(struct ieee80211_local *local, int _idx,
                 struct survey_info *survey),

        TP_ARGS(local, _idx, survey),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(int, idx)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->idx = _idx;
        ),

        TP_printk(
                LOCAL_PR_FMT " idx:%d",
                LOCAL_PR_ARG, __entry->idx
        )
);

TRACE_EVENT(drv_flush,
        TP_PROTO(struct ieee80211_local *local,
                 u32 queues, bool drop),

        TP_ARGS(local, queues, drop),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(bool, drop)
                __field(u32, queues)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->drop = drop;
                __entry->queues = queues;
        ),

        TP_printk(
                LOCAL_PR_FMT " queues:0x%x drop:%d",
                LOCAL_PR_ARG, __entry->queues, __entry->drop
        )
);

DEFINE_EVENT(sta_event, drv_flush_sta,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),
        TP_ARGS(local, sdata, sta)
);

DECLARE_EVENT_CLASS(chanswitch_evt,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_channel_switch *ch_switch),

        TP_ARGS(local, sdata, ch_switch),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                CHANDEF_ENTRY
                __field(u64, timestamp)
                __field(u32, device_timestamp)
                __field(bool, block_tx)
                __field(u8, count)
                __field(u8, link_id)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                CHANDEF_ASSIGN(&ch_switch->chandef)
                __entry->timestamp = ch_switch->timestamp;
                __entry->device_timestamp = ch_switch->device_timestamp;
                __entry->block_tx = ch_switch->block_tx;
                __entry->count = ch_switch->count;
                __entry->link_id = ch_switch->link_id;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT CHANDEF_PR_FMT  " count:%d block_tx:%d timestamp:%llu device_ts:%u link_id:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, CHANDEF_PR_ARG, __entry->count,
                __entry->block_tx, __entry->timestamp,
                __entry->device_timestamp, __entry->link_id
        )
);

DEFINE_EVENT(chanswitch_evt, drv_channel_switch,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_channel_switch *ch_switch),
        TP_ARGS(local, sdata, ch_switch)
);

TRACE_EVENT(drv_set_antenna,
        TP_PROTO(struct ieee80211_local *local, u32 tx_ant, u32 rx_ant, int ret),

        TP_ARGS(local, tx_ant, rx_ant, ret),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u32, tx_ant)
                __field(u32, rx_ant)
                __field(int, ret)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->tx_ant = tx_ant;
                __entry->rx_ant = rx_ant;
                __entry->ret = ret;
        ),

        TP_printk(
                LOCAL_PR_FMT " tx_ant:%d rx_ant:%d ret:%d",
                LOCAL_PR_ARG, __entry->tx_ant, __entry->rx_ant, __entry->ret
        )
);

TRACE_EVENT(drv_get_antenna,
        TP_PROTO(struct ieee80211_local *local, u32 tx_ant, u32 rx_ant, int ret),

        TP_ARGS(local, tx_ant, rx_ant, ret),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u32, tx_ant)
                __field(u32, rx_ant)
                __field(int, ret)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->tx_ant = tx_ant;
                __entry->rx_ant = rx_ant;
                __entry->ret = ret;
        ),

        TP_printk(
                LOCAL_PR_FMT " tx_ant:%d rx_ant:%d ret:%d",
                LOCAL_PR_ARG, __entry->tx_ant, __entry->rx_ant, __entry->ret
        )
);

TRACE_EVENT(drv_remain_on_channel,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_channel *chan,
                 unsigned int duration,
                 enum ieee80211_roc_type type),

        TP_ARGS(local, sdata, chan, duration, type),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(int, center_freq)
                __field(int, freq_offset)
                __field(unsigned int, duration)
                __field(u32, type)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->center_freq = chan->center_freq;
                __entry->freq_offset = chan->freq_offset;
                __entry->duration = duration;
                __entry->type = type;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT " freq:%d.%03dMHz duration:%dms type=%d",
                LOCAL_PR_ARG, VIF_PR_ARG,
                __entry->center_freq, __entry->freq_offset,
                __entry->duration, __entry->type
        )
);

DEFINE_EVENT(local_sdata_evt, drv_cancel_remain_on_channel,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

TRACE_EVENT(drv_set_ringparam,
        TP_PROTO(struct ieee80211_local *local, u32 tx, u32 rx),

        TP_ARGS(local, tx, rx),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u32, tx)
                __field(u32, rx)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->tx = tx;
                __entry->rx = rx;
        ),

        TP_printk(
                LOCAL_PR_FMT " tx:%d rx %d",
                LOCAL_PR_ARG, __entry->tx, __entry->rx
        )
);

TRACE_EVENT(drv_get_ringparam,
        TP_PROTO(struct ieee80211_local *local, u32 *tx, u32 *tx_max,
                 u32 *rx, u32 *rx_max),

        TP_ARGS(local, tx, tx_max, rx, rx_max),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u32, tx)
                __field(u32, tx_max)
                __field(u32, rx)
                __field(u32, rx_max)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->tx = *tx;
                __entry->tx_max = *tx_max;
                __entry->rx = *rx;
                __entry->rx_max = *rx_max;
        ),

        TP_printk(
                LOCAL_PR_FMT " tx:%d tx_max %d rx %d rx_max %d",
                LOCAL_PR_ARG,
                __entry->tx, __entry->tx_max, __entry->rx, __entry->rx_max
        )
);

DEFINE_EVENT(local_only_evt, drv_tx_frames_pending,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

DEFINE_EVENT(local_only_evt, drv_offchannel_tx_cancel_wait,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

TRACE_EVENT(drv_set_bitrate_mask,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 const struct cfg80211_bitrate_mask *mask),

        TP_ARGS(local, sdata, mask),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u32, legacy_2g)
                __field(u32, legacy_5g)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->legacy_2g = mask->control[NL80211_BAND_2GHZ].legacy;
                __entry->legacy_5g = mask->control[NL80211_BAND_5GHZ].legacy;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT " 2G Mask:0x%x 5G Mask:0x%x",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->legacy_2g, __entry->legacy_5g
        )
);

TRACE_EVENT(drv_set_rekey_data,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct cfg80211_gtk_rekey_data *data),

        TP_ARGS(local, sdata, data),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __array(u8, kek, NL80211_KEK_LEN)
                __array(u8, kck, NL80211_KCK_LEN)
                __array(u8, replay_ctr, NL80211_REPLAY_CTR_LEN)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                memcpy(__entry->kek, data->kek, NL80211_KEK_LEN);
                memcpy(__entry->kck, data->kck, NL80211_KCK_LEN);
                memcpy(__entry->replay_ctr, data->replay_ctr,
                       NL80211_REPLAY_CTR_LEN);
        ),

        TP_printk(LOCAL_PR_FMT VIF_PR_FMT,
                  LOCAL_PR_ARG, VIF_PR_ARG)
);

TRACE_EVENT(drv_event_callback,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 const struct ieee80211_event *_event),

        TP_ARGS(local, sdata, _event),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u32, type)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->type = _event->type;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " event:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->type
        )
);

DECLARE_EVENT_CLASS(release_evt,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta,
                 u16 tids, int num_frames,
                 enum ieee80211_frame_release_type reason,
                 bool more_data),

        TP_ARGS(local, sta, tids, num_frames, reason, more_data),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                STA_ENTRY
                __field(u16, tids)
                __field(int, num_frames)
                __field(int, reason)
                __field(bool, more_data)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                STA_ASSIGN;
                __entry->tids = tids;
                __entry->num_frames = num_frames;
                __entry->reason = reason;
                __entry->more_data = more_data;
        ),

        TP_printk(
                LOCAL_PR_FMT STA_PR_FMT
                " TIDs:0x%.4x frames:%d reason:%d more:%d",
                LOCAL_PR_ARG, STA_PR_ARG, __entry->tids, __entry->num_frames,
                __entry->reason, __entry->more_data
        )
);

DEFINE_EVENT(release_evt, drv_release_buffered_frames,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta,
                 u16 tids, int num_frames,
                 enum ieee80211_frame_release_type reason,
                 bool more_data),

        TP_ARGS(local, sta, tids, num_frames, reason, more_data)
);

DEFINE_EVENT(release_evt, drv_allow_buffered_frames,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta,
                 u16 tids, int num_frames,
                 enum ieee80211_frame_release_type reason,
                 bool more_data),

        TP_ARGS(local, sta, tids, num_frames, reason, more_data)
);

DECLARE_EVENT_CLASS(mgd_prepare_complete_tx_evt,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 u16 duration, u16 subtype, bool success),

        TP_ARGS(local, sdata, duration, subtype, success),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u32, duration)
                __field(u16, subtype)
                __field(u8, success)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->duration = duration;
                __entry->subtype = subtype;
                __entry->success = success;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " duration: %u, subtype:0x%x, success:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->duration,
                __entry->subtype, __entry->success
        )
);

DEFINE_EVENT(mgd_prepare_complete_tx_evt, drv_mgd_prepare_tx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 u16 duration, u16 subtype, bool success),

        TP_ARGS(local, sdata, duration, subtype, success)
);

DEFINE_EVENT(mgd_prepare_complete_tx_evt, drv_mgd_complete_tx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 u16 duration, u16 subtype, bool success),

        TP_ARGS(local, sdata, duration, subtype, success)
);

DEFINE_EVENT(local_sdata_evt, drv_mgd_protect_tdls_discover,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),

        TP_ARGS(local, sdata)
);

DECLARE_EVENT_CLASS(local_chanctx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_chanctx *ctx),

        TP_ARGS(local, ctx),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                CHANCTX_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                CHANCTX_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT CHANCTX_PR_FMT,
                LOCAL_PR_ARG, CHANCTX_PR_ARG
        )
);

DEFINE_EVENT(local_chanctx, drv_add_chanctx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_chanctx *ctx),
        TP_ARGS(local, ctx)
);

DEFINE_EVENT(local_chanctx, drv_remove_chanctx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_chanctx *ctx),
        TP_ARGS(local, ctx)
);

TRACE_EVENT(drv_change_chanctx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_chanctx *ctx,
                 u32 changed),

        TP_ARGS(local, ctx, changed),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                CHANCTX_ENTRY
                __field(u32, changed)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                CHANCTX_ASSIGN;
                __entry->changed = changed;
        ),

        TP_printk(
                LOCAL_PR_FMT CHANCTX_PR_FMT " changed:%#x",
                LOCAL_PR_ARG, CHANCTX_PR_ARG, __entry->changed
        )
);

#if !defined(__TRACE_VIF_ENTRY)
#define __TRACE_VIF_ENTRY
struct trace_vif_entry {
        enum nl80211_iftype vif_type;
        bool p2p;
        char vif_name[IFNAMSIZ];
} __packed;

struct trace_chandef_entry {
        u32 control_freq;
        u32 freq_offset;
        u32 chan_width;
        u32 center_freq1;
        u32 freq1_offset;
        u32 center_freq2;
} __packed;

struct trace_switch_entry {
        struct trace_vif_entry vif;
        unsigned int link_id;
        struct trace_chandef_entry old_chandef;
        struct trace_chandef_entry new_chandef;
} __packed;

#define SWITCH_ENTRY_ASSIGN(to, from) local_vifs[i].to = vifs[i].from
#endif

TRACE_EVENT(drv_switch_vif_chanctx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_vif_chanctx_switch *vifs,
                 int n_vifs, enum ieee80211_chanctx_switch_mode mode),
            TP_ARGS(local, vifs, n_vifs, mode),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(int, n_vifs)
                __field(u32, mode)
                __dynamic_array(u8, vifs,
                                sizeof(struct trace_switch_entry) * n_vifs)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->n_vifs = n_vifs;
                __entry->mode = mode;
                {
                        struct trace_switch_entry *local_vifs =
                                __get_dynamic_array(vifs);
                        int i;

                        for (i = 0; i < n_vifs; i++) {
                                struct ieee80211_sub_if_data *sdata;

                                sdata = container_of(vifs[i].vif,
                                                struct ieee80211_sub_if_data,
                                                vif);

                                SWITCH_ENTRY_ASSIGN(vif.vif_type, vif->type);
                                SWITCH_ENTRY_ASSIGN(vif.p2p, vif->p2p);
                                SWITCH_ENTRY_ASSIGN(link_id, link_conf->link_id);
                                strncpy(local_vifs[i].vif.vif_name,
                                        sdata->name,
                                        sizeof(local_vifs[i].vif.vif_name));
                                SWITCH_ENTRY_ASSIGN(old_chandef.control_freq,
                                                old_ctx->def.chan->center_freq);
                                SWITCH_ENTRY_ASSIGN(old_chandef.freq_offset,
                                                old_ctx->def.chan->freq_offset);
                                SWITCH_ENTRY_ASSIGN(old_chandef.chan_width,
                                                    old_ctx->def.width);
                                SWITCH_ENTRY_ASSIGN(old_chandef.center_freq1,
                                                    old_ctx->def.center_freq1);
                                SWITCH_ENTRY_ASSIGN(old_chandef.freq1_offset,
                                                    old_ctx->def.freq1_offset);
                                SWITCH_ENTRY_ASSIGN(old_chandef.center_freq2,
                                                    old_ctx->def.center_freq2);
                                SWITCH_ENTRY_ASSIGN(new_chandef.control_freq,
                                                new_ctx->def.chan->center_freq);
                                SWITCH_ENTRY_ASSIGN(new_chandef.freq_offset,
                                                new_ctx->def.chan->freq_offset);
                                SWITCH_ENTRY_ASSIGN(new_chandef.chan_width,
                                                    new_ctx->def.width);
                                SWITCH_ENTRY_ASSIGN(new_chandef.center_freq1,
                                                    new_ctx->def.center_freq1);
                                SWITCH_ENTRY_ASSIGN(new_chandef.freq1_offset,
                                                    new_ctx->def.freq1_offset);
                                SWITCH_ENTRY_ASSIGN(new_chandef.center_freq2,
                                                    new_ctx->def.center_freq2);
                        }
                }
        ),

        TP_printk(
                LOCAL_PR_FMT " n_vifs:%d mode:%d",
                LOCAL_PR_ARG, __entry->n_vifs, __entry->mode
        )
);

DECLARE_EVENT_CLASS(local_sdata_chanctx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_bss_conf *link_conf,
                 struct ieee80211_chanctx *ctx),

        TP_ARGS(local, sdata, link_conf, ctx),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                CHANCTX_ENTRY
                __field(unsigned int, link_id)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                CHANCTX_ASSIGN;
                __entry->link_id = link_conf->link_id;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " link_id:%d" CHANCTX_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->link_id, CHANCTX_PR_ARG
        )
);

DEFINE_EVENT(local_sdata_chanctx, drv_assign_vif_chanctx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_bss_conf *link_conf,
                 struct ieee80211_chanctx *ctx),
        TP_ARGS(local, sdata, link_conf, ctx)
);

DEFINE_EVENT(local_sdata_chanctx, drv_unassign_vif_chanctx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_bss_conf *link_conf,
                 struct ieee80211_chanctx *ctx),
        TP_ARGS(local, sdata, link_conf, ctx)
);

TRACE_EVENT(drv_start_ap,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_bss_conf *link_conf),

        TP_ARGS(local, sdata, link_conf),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u32, link_id)
                __field(u8, dtimper)
                __field(u16, bcnint)
                __dynamic_array(u8, ssid, sdata->vif.cfg.ssid_len)
                __field(bool, hidden_ssid)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->link_id = link_conf->link_id;
                __entry->dtimper = link_conf->dtim_period;
                __entry->bcnint = link_conf->beacon_int;
                __entry->hidden_ssid = link_conf->hidden_ssid;
                memcpy(__get_dynamic_array(ssid),
                       sdata->vif.cfg.ssid,
                       sdata->vif.cfg.ssid_len);
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT " link id %u",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->link_id
        )
);

TRACE_EVENT(drv_stop_ap,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_bss_conf *link_conf),

        TP_ARGS(local, sdata, link_conf),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u32, link_id)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->link_id = link_conf->link_id;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT " link id %u",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->link_id
        )
);

TRACE_EVENT(drv_reconfig_complete,
        TP_PROTO(struct ieee80211_local *local,
                 enum ieee80211_reconfig_type reconfig_type),
        TP_ARGS(local, reconfig_type),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u8, reconfig_type)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->reconfig_type = reconfig_type;
        ),

        TP_printk(
                LOCAL_PR_FMT  " reconfig_type:%d",
                LOCAL_PR_ARG, __entry->reconfig_type
        )

);

#if IS_ENABLED(CONFIG_IPV6)
DEFINE_EVENT(local_sdata_evt, drv_ipv6_addr_change,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);
#endif

TRACE_EVENT(drv_join_ibss,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_bss_conf *info),

        TP_ARGS(local, sdata, info),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u8, dtimper)
                __field(u16, bcnint)
                __dynamic_array(u8, ssid, sdata->vif.cfg.ssid_len)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->dtimper = info->dtim_period;
                __entry->bcnint = info->beacon_int;
                memcpy(__get_dynamic_array(ssid),
                       sdata->vif.cfg.ssid,
                       sdata->vif.cfg.ssid_len);
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG
        )
);

DEFINE_EVENT(local_sdata_evt, drv_leave_ibss,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

TRACE_EVENT(drv_get_expected_throughput,
        TP_PROTO(struct ieee80211_sta *sta),

        TP_ARGS(sta),

        TP_STRUCT__entry(
                STA_ENTRY
        ),

        TP_fast_assign(
                STA_ASSIGN;
        ),

        TP_printk(
                STA_PR_FMT, STA_PR_ARG
        )
);

TRACE_EVENT(drv_start_nan,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct cfg80211_nan_conf *conf),

        TP_ARGS(local, sdata, conf),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u8, master_pref)
                __field(u8, bands)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->master_pref = conf->master_pref;
                __entry->bands = conf->bands;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT
                ", master preference: %u, bands: 0x%0x",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->master_pref,
                __entry->bands
        )
);

TRACE_EVENT(drv_stop_nan,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),

        TP_ARGS(local, sdata),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG
        )
);

TRACE_EVENT(drv_nan_change_conf,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct cfg80211_nan_conf *conf,
                 u32 changes),

        TP_ARGS(local, sdata, conf, changes),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u8, master_pref)
                __field(u8, bands)
                __field(u32, changes)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->master_pref = conf->master_pref;
                __entry->bands = conf->bands;
                __entry->changes = changes;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT
                ", master preference: %u, bands: 0x%0x, changes: 0x%x",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->master_pref,
                __entry->bands, __entry->changes
        )
);

TRACE_EVENT(drv_add_nan_func,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 const struct cfg80211_nan_func *func),

        TP_ARGS(local, sdata, func),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u8, type)
                __field(u8, inst_id)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->type = func->type;
                __entry->inst_id = func->instance_id;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT
                ", type: %u, inst_id: %u",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->type, __entry->inst_id
        )
);

TRACE_EVENT(drv_del_nan_func,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 u8 instance_id),

        TP_ARGS(local, sdata, instance_id),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u8, instance_id)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->instance_id = instance_id;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT
                ", instance_id: %u",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->instance_id
        )
);

DEFINE_EVENT(local_sdata_evt, drv_start_pmsr,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

DEFINE_EVENT(local_sdata_evt, drv_abort_pmsr,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

TRACE_EVENT(drv_set_default_unicast_key,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 int key_idx),

        TP_ARGS(local, sdata, key_idx),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(int, key_idx)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->key_idx = key_idx;
        ),

        TP_printk(LOCAL_PR_FMT VIF_PR_FMT " key_idx:%d",
                  LOCAL_PR_ARG, VIF_PR_ARG, __entry->key_idx)
);

TRACE_EVENT(drv_channel_switch_beacon,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct cfg80211_chan_def *chandef),

        TP_ARGS(local, sdata, chandef),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                CHANDEF_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                CHANDEF_ASSIGN(chandef);
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " channel switch to " CHANDEF_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG, CHANDEF_PR_ARG
        )
);

DEFINE_EVENT(chanswitch_evt, drv_pre_channel_switch,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_channel_switch *ch_switch),
        TP_ARGS(local, sdata, ch_switch)
);

DEFINE_EVENT(local_sdata_evt, drv_post_channel_switch,
             TP_PROTO(struct ieee80211_local *local,
                      struct ieee80211_sub_if_data *sdata),
             TP_ARGS(local, sdata)
);

DEFINE_EVENT(local_sdata_evt, drv_abort_channel_switch,
             TP_PROTO(struct ieee80211_local *local,
                      struct ieee80211_sub_if_data *sdata),
             TP_ARGS(local, sdata)
);

DEFINE_EVENT(chanswitch_evt, drv_channel_switch_rx_beacon,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_channel_switch *ch_switch),
        TP_ARGS(local, sdata, ch_switch)
);

TRACE_EVENT(drv_get_txpower,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 unsigned int link_id, int dbm, int ret),

        TP_ARGS(local, sdata, link_id, dbm, ret),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(unsigned int, link_id)
                __field(int, dbm)
                __field(int, ret)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->link_id = link_id;
                __entry->dbm = dbm;
                __entry->ret = ret;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " link_id:%d dbm:%d ret:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->link_id, __entry->dbm, __entry->ret
        )
);

TRACE_EVENT(drv_tdls_channel_switch,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta, u8 oper_class,
                 struct cfg80211_chan_def *chandef),

        TP_ARGS(local, sdata, sta, oper_class, chandef),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(u8, oper_class)
                CHANDEF_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
                __entry->oper_class = oper_class;
                CHANDEF_ASSIGN(chandef)
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " tdls channel switch to"
                CHANDEF_PR_FMT  " oper_class:%d " STA_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG, CHANDEF_PR_ARG, __entry->oper_class,
                STA_PR_ARG
        )
);

TRACE_EVENT(drv_tdls_cancel_channel_switch,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),

        TP_ARGS(local, sdata, sta),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT
                " tdls cancel channel switch with " STA_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG
        )
);

TRACE_EVENT(drv_tdls_recv_channel_switch,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_tdls_ch_sw_params *params),

        TP_ARGS(local, sdata, params),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u8, action_code)
                STA_ENTRY
                CHANDEF_ENTRY
                __field(u32, status)
                __field(bool, peer_initiator)
                __field(u32, timestamp)
                __field(u16, switch_time)
                __field(u16, switch_timeout)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_NAMED_ASSIGN(params->sta);
                CHANDEF_ASSIGN(params->chandef)
                __entry->peer_initiator = params->sta->tdls_initiator;
                __entry->action_code = params->action_code;
                __entry->status = params->status;
                __entry->timestamp = params->timestamp;
                __entry->switch_time = params->switch_time;
                __entry->switch_timeout = params->switch_timeout;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " received tdls channel switch packet"
                " action:%d status:%d time:%d switch time:%d switch"
                " timeout:%d initiator: %d chan:" CHANDEF_PR_FMT STA_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->action_code, __entry->status,
                __entry->timestamp, __entry->switch_time,
                __entry->switch_timeout, __entry->peer_initiator,
                CHANDEF_PR_ARG, STA_PR_ARG
        )
);

TRACE_EVENT(drv_wake_tx_queue,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct txq_info *txq),

        TP_ARGS(local, sdata, txq),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(u8, ac)
                __field(u8, tid)
        ),

        TP_fast_assign(
                struct ieee80211_sta *sta = txq->txq.sta;

                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
                __entry->ac = txq->txq.ac;
                __entry->tid = txq->txq.tid;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  STA_PR_FMT " ac:%d tid:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG, __entry->ac, __entry->tid
        )
);

TRACE_EVENT(drv_get_ftm_responder_stats,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct cfg80211_ftm_responder_stats *ftm_stats),

        TP_ARGS(local, sdata, ftm_stats),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG
        )
);

DEFINE_EVENT(local_sdata_addr_evt, drv_update_vif_offload,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

DECLARE_EVENT_CLASS(sta_flag_evt,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta, bool enabled),

        TP_ARGS(local, sdata, sta, enabled),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(bool, enabled)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
                __entry->enabled = enabled;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  STA_PR_FMT " enabled:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG, __entry->enabled
        )
);

DEFINE_EVENT(sta_flag_evt, drv_sta_set_4addr,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta, bool enabled),

        TP_ARGS(local, sdata, sta, enabled)
);

DEFINE_EVENT(sta_flag_evt, drv_sta_set_decap_offload,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta, bool enabled),

        TP_ARGS(local, sdata, sta, enabled)
);

TRACE_EVENT(drv_add_twt_setup,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta,
                 struct ieee80211_twt_setup *twt,
                 struct ieee80211_twt_params *twt_agrt),

        TP_ARGS(local, sta, twt, twt_agrt),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                STA_ENTRY
                __field(u8, dialog_token)
                __field(u8, control)
                __field(__le16, req_type)
                __field(__le64, twt)
                __field(u8, duration)
                __field(__le16, mantissa)
                __field(u8, channel)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                STA_ASSIGN;
                __entry->dialog_token = twt->dialog_token;
                __entry->control = twt->control;
                __entry->req_type = twt_agrt->req_type;
                __entry->twt = twt_agrt->twt;
                __entry->duration = twt_agrt->min_twt_dur;
                __entry->mantissa = twt_agrt->mantissa;
                __entry->channel = twt_agrt->channel;
        ),

        TP_printk(
                LOCAL_PR_FMT STA_PR_FMT
                " token:%d control:0x%02x req_type:0x%04x"
                " twt:%llu duration:%d mantissa:%d channel:%d",
                LOCAL_PR_ARG, STA_PR_ARG, __entry->dialog_token,
                __entry->control, le16_to_cpu(__entry->req_type),
                le64_to_cpu(__entry->twt), __entry->duration,
                le16_to_cpu(__entry->mantissa), __entry->channel
        )
);

TRACE_EVENT(drv_twt_teardown_request,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta, u8 flowid),

        TP_ARGS(local, sta, flowid),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                STA_ENTRY
                __field(u8, flowid)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                STA_ASSIGN;
                __entry->flowid = flowid;
        ),

        TP_printk(
                LOCAL_PR_FMT STA_PR_FMT " flowid:%d",
                LOCAL_PR_ARG, STA_PR_ARG, __entry->flowid
        )
);

DEFINE_EVENT(sta_event, drv_net_fill_forward_path,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),
        TP_ARGS(local, sdata, sta)
);

TRACE_EVENT(drv_net_setup_tc,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 u8 type),

        TP_ARGS(local, sdata, type),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u8, type)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->type = type;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " type:%d\n",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->type
        )
);

TRACE_EVENT(drv_can_activate_links,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 u16 active_links),

        TP_ARGS(local, sdata, active_links),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u16, active_links)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->active_links = active_links;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " requested active_links:0x%04x\n",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->active_links
        )
);

TRACE_EVENT(drv_change_vif_links,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 u16 old_links, u16 new_links),

        TP_ARGS(local, sdata, old_links, new_links),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u16, old_links)
                __field(u16, new_links)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->old_links = old_links;
                __entry->new_links = new_links;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " old_links:0x%04x, new_links:0x%04x\n",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->old_links, __entry->new_links
        )
);

TRACE_EVENT(drv_change_sta_links,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta,
                 u16 old_links, u16 new_links),

        TP_ARGS(local, sdata, sta, old_links, new_links),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(u16, old_links)
                __field(u16, new_links)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
                __entry->old_links = old_links;
                __entry->new_links = new_links;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT STA_PR_FMT " old_links:0x%04x, new_links:0x%04x\n",
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG,
                __entry->old_links, __entry->new_links
        )
);

/*
 * Tracing for API calls that drivers call.
 */

TRACE_EVENT(api_return_bool,
        TP_PROTO(struct ieee80211_local *local, bool result),

        TP_ARGS(local, result),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(bool, result)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->result = result;
        ),

        TP_printk(
                LOCAL_PR_FMT " result=%d",
                LOCAL_PR_ARG, __entry->result
        )
);

TRACE_EVENT(api_return_void,
        TP_PROTO(struct ieee80211_local *local),

        TP_ARGS(local),

        TP_STRUCT__entry(
                LOCAL_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT, LOCAL_PR_ARG
        )
);

TRACE_EVENT(api_start_tx_ba_session,
        TP_PROTO(struct ieee80211_sta *sta, u16 tid),

        TP_ARGS(sta, tid),

        TP_STRUCT__entry(
                STA_ENTRY
                __field(u16, tid)
        ),

        TP_fast_assign(
                STA_ASSIGN;
                __entry->tid = tid;
        ),

        TP_printk(
                STA_PR_FMT " tid:%d",
                STA_PR_ARG, __entry->tid
        )
);

TRACE_EVENT(api_start_tx_ba_cb,
        TP_PROTO(struct ieee80211_sub_if_data *sdata, const u8 *ra, u16 tid),

        TP_ARGS(sdata, ra, tid),

        TP_STRUCT__entry(
                VIF_ENTRY
                __array(u8, ra, ETH_ALEN)
                __field(u16, tid)
        ),

        TP_fast_assign(
                VIF_ASSIGN;
                memcpy(__entry->ra, ra, ETH_ALEN);
                __entry->tid = tid;
        ),

        TP_printk(
                VIF_PR_FMT " ra:%pM tid:%d",
                VIF_PR_ARG, __entry->ra, __entry->tid
        )
);

TRACE_EVENT(api_stop_tx_ba_session,
        TP_PROTO(struct ieee80211_sta *sta, u16 tid),

        TP_ARGS(sta, tid),

        TP_STRUCT__entry(
                STA_ENTRY
                __field(u16, tid)
        ),

        TP_fast_assign(
                STA_ASSIGN;
                __entry->tid = tid;
        ),

        TP_printk(
                STA_PR_FMT " tid:%d",
                STA_PR_ARG, __entry->tid
        )
);

TRACE_EVENT(api_stop_tx_ba_cb,
        TP_PROTO(struct ieee80211_sub_if_data *sdata, const u8 *ra, u16 tid),

        TP_ARGS(sdata, ra, tid),

        TP_STRUCT__entry(
                VIF_ENTRY
                __array(u8, ra, ETH_ALEN)
                __field(u16, tid)
        ),

        TP_fast_assign(
                VIF_ASSIGN;
                memcpy(__entry->ra, ra, ETH_ALEN);
                __entry->tid = tid;
        ),

        TP_printk(
                VIF_PR_FMT " ra:%pM tid:%d",
                VIF_PR_ARG, __entry->ra, __entry->tid
        )
);

DEFINE_EVENT(local_only_evt, api_restart_hw,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

TRACE_EVENT(api_beacon_loss,
        TP_PROTO(struct ieee80211_sub_if_data *sdata),

        TP_ARGS(sdata),

        TP_STRUCT__entry(
                VIF_ENTRY
        ),

        TP_fast_assign(
                VIF_ASSIGN;
        ),

        TP_printk(
                VIF_PR_FMT,
                VIF_PR_ARG
        )
);

TRACE_EVENT(api_connection_loss,
        TP_PROTO(struct ieee80211_sub_if_data *sdata),

        TP_ARGS(sdata),

        TP_STRUCT__entry(
                VIF_ENTRY
        ),

        TP_fast_assign(
                VIF_ASSIGN;
        ),

        TP_printk(
                VIF_PR_FMT,
                VIF_PR_ARG
        )
);

TRACE_EVENT(api_disconnect,
        TP_PROTO(struct ieee80211_sub_if_data *sdata, bool reconnect),

        TP_ARGS(sdata, reconnect),

        TP_STRUCT__entry(
                VIF_ENTRY
                __field(int, reconnect)
        ),

        TP_fast_assign(
                VIF_ASSIGN;
                __entry->reconnect = reconnect;
        ),

        TP_printk(
                VIF_PR_FMT " reconnect:%d",
                VIF_PR_ARG, __entry->reconnect
        )
);

TRACE_EVENT(api_cqm_rssi_notify,
        TP_PROTO(struct ieee80211_sub_if_data *sdata,
                 enum nl80211_cqm_rssi_threshold_event rssi_event,
                 s32 rssi_level),

        TP_ARGS(sdata, rssi_event, rssi_level),

        TP_STRUCT__entry(
                VIF_ENTRY
                __field(u32, rssi_event)
                __field(s32, rssi_level)
        ),

        TP_fast_assign(
                VIF_ASSIGN;
                __entry->rssi_event = rssi_event;
                __entry->rssi_level = rssi_level;
        ),

        TP_printk(
                VIF_PR_FMT " event:%d rssi:%d",
                VIF_PR_ARG, __entry->rssi_event, __entry->rssi_level
        )
);

DEFINE_EVENT(local_sdata_evt, api_cqm_beacon_loss_notify,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

TRACE_EVENT(api_scan_completed,
        TP_PROTO(struct ieee80211_local *local, bool aborted),

        TP_ARGS(local, aborted),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(bool, aborted)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->aborted = aborted;
        ),

        TP_printk(
                LOCAL_PR_FMT " aborted:%d",
                LOCAL_PR_ARG, __entry->aborted
        )
);

TRACE_EVENT(api_sched_scan_results,
        TP_PROTO(struct ieee80211_local *local),

        TP_ARGS(local),

        TP_STRUCT__entry(
                LOCAL_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT, LOCAL_PR_ARG
        )
);

TRACE_EVENT(api_sched_scan_stopped,
        TP_PROTO(struct ieee80211_local *local),

        TP_ARGS(local),

        TP_STRUCT__entry(
                LOCAL_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT, LOCAL_PR_ARG
        )
);

TRACE_EVENT(api_sta_block_awake,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta, bool block),

        TP_ARGS(local, sta, block),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                STA_ENTRY
                __field(bool, block)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                STA_ASSIGN;
                __entry->block = block;
        ),

        TP_printk(
                LOCAL_PR_FMT STA_PR_FMT " block:%d",
                LOCAL_PR_ARG, STA_PR_ARG, __entry->block
        )
);

TRACE_EVENT(api_chswitch_done,
        TP_PROTO(struct ieee80211_sub_if_data *sdata, bool success,
                 unsigned int link_id),

        TP_ARGS(sdata, success, link_id),

        TP_STRUCT__entry(
                VIF_ENTRY
                __field(bool, success)
                __field(unsigned int, link_id)
        ),

        TP_fast_assign(
                VIF_ASSIGN;
                __entry->success = success;
                __entry->link_id = link_id;
        ),

        TP_printk(
                VIF_PR_FMT " success=%d link_id=%d",
                VIF_PR_ARG, __entry->success, __entry->link_id
        )
);

DEFINE_EVENT(local_only_evt, api_ready_on_channel,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

DEFINE_EVENT(local_only_evt, api_remain_on_channel_expired,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

TRACE_EVENT(api_gtk_rekey_notify,
        TP_PROTO(struct ieee80211_sub_if_data *sdata,
                 const u8 *bssid, const u8 *replay_ctr),

        TP_ARGS(sdata, bssid, replay_ctr),

        TP_STRUCT__entry(
                VIF_ENTRY
                __array(u8, bssid, ETH_ALEN)
                __array(u8, replay_ctr, NL80211_REPLAY_CTR_LEN)
        ),

        TP_fast_assign(
                VIF_ASSIGN;
                memcpy(__entry->bssid, bssid, ETH_ALEN);
                memcpy(__entry->replay_ctr, replay_ctr, NL80211_REPLAY_CTR_LEN);
        ),

        TP_printk(VIF_PR_FMT, VIF_PR_ARG)
);

TRACE_EVENT(api_enable_rssi_reports,
        TP_PROTO(struct ieee80211_sub_if_data *sdata,
                 int rssi_min_thold, int rssi_max_thold),

        TP_ARGS(sdata, rssi_min_thold, rssi_max_thold),

        TP_STRUCT__entry(
                VIF_ENTRY
                __field(int, rssi_min_thold)
                __field(int, rssi_max_thold)
        ),

        TP_fast_assign(
                VIF_ASSIGN;
                __entry->rssi_min_thold = rssi_min_thold;
                __entry->rssi_max_thold = rssi_max_thold;
        ),

        TP_printk(
                VIF_PR_FMT " rssi_min_thold =%d, rssi_max_thold = %d",
                VIF_PR_ARG, __entry->rssi_min_thold, __entry->rssi_max_thold
        )
);

TRACE_EVENT(api_eosp,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta),

        TP_ARGS(local, sta),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                STA_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                STA_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT STA_PR_FMT,
                LOCAL_PR_ARG, STA_PR_ARG
        )
);

TRACE_EVENT(api_send_eosp_nullfunc,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta,
                 u8 tid),

        TP_ARGS(local, sta, tid),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                STA_ENTRY
                __field(u8, tid)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                STA_ASSIGN;
                __entry->tid = tid;
        ),

        TP_printk(
                LOCAL_PR_FMT STA_PR_FMT " tid:%d",
                LOCAL_PR_ARG, STA_PR_ARG, __entry->tid
        )
);

TRACE_EVENT(api_sta_set_buffered,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta,
                 u8 tid, bool buffered),

        TP_ARGS(local, sta, tid, buffered),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                STA_ENTRY
                __field(u8, tid)
                __field(bool, buffered)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                STA_ASSIGN;
                __entry->tid = tid;
                __entry->buffered = buffered;
        ),

        TP_printk(
                LOCAL_PR_FMT STA_PR_FMT " tid:%d buffered:%d",
                LOCAL_PR_ARG, STA_PR_ARG, __entry->tid, __entry->buffered
        )
);

TRACE_EVENT(api_radar_detected,
        TP_PROTO(struct ieee80211_local *local),

        TP_ARGS(local),

        TP_STRUCT__entry(
                LOCAL_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT " radar detected",
                LOCAL_PR_ARG
        )
);

TRACE_EVENT(api_request_smps,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_link_data *link,
                 enum ieee80211_smps_mode smps_mode),

        TP_ARGS(local, sdata, link, smps_mode),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(int, link_id)
                __field(u32, smps_mode)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->link_id = link->link_id,
                __entry->smps_mode = smps_mode;
        ),

        TP_printk(
                LOCAL_PR_FMT " " VIF_PR_FMT " link:%d, smps_mode:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->link_id, __entry->smps_mode
        )
);

TRACE_EVENT(api_prepare_rx_omi_bw,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct link_sta_info *link_sta,
                 enum ieee80211_sta_rx_bandwidth bw),

        TP_ARGS(local, sdata, link_sta, bw),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(int, link_id)
                __field(u32, bw)
                __field(bool, result)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_NAMED_ASSIGN(link_sta->sta);
                __entry->link_id = link_sta->link_id;
                __entry->bw = bw;
        ),

        TP_printk(
                LOCAL_PR_FMT " " VIF_PR_FMT " " STA_PR_FMT " link:%d, bw:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG,
                __entry->link_id, __entry->bw
        )
);

TRACE_EVENT(api_finalize_rx_omi_bw,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct link_sta_info *link_sta),

        TP_ARGS(local, sdata, link_sta),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(int, link_id)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_NAMED_ASSIGN(link_sta->sta);
                __entry->link_id = link_sta->link_id;
        ),

        TP_printk(
                LOCAL_PR_FMT " " VIF_PR_FMT " " STA_PR_FMT " link:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG, __entry->link_id
        )
);

/*
 * Tracing for internal functions
 * (which may also be called in response to driver calls)
 */

TRACE_EVENT(wake_queue,
        TP_PROTO(struct ieee80211_local *local, u16 queue,
                 enum queue_stop_reason reason, int refcount),

        TP_ARGS(local, queue, reason, refcount),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u16, queue)
                __field(u32, reason)
                __field(int, refcount)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->queue = queue;
                __entry->reason = reason;
                __entry->refcount = refcount;
        ),

        TP_printk(
                LOCAL_PR_FMT " queue:%d, reason:%d, refcount: %d",
                LOCAL_PR_ARG, __entry->queue, __entry->reason,
                __entry->refcount
        )
);

TRACE_EVENT(stop_queue,
        TP_PROTO(struct ieee80211_local *local, u16 queue,
                 enum queue_stop_reason reason, int refcount),

        TP_ARGS(local, queue, reason, refcount),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u16, queue)
                __field(u32, reason)
                __field(int, refcount)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->queue = queue;
                __entry->reason = reason;
                __entry->refcount = refcount;
        ),

        TP_printk(
                LOCAL_PR_FMT " queue:%d, reason:%d, refcount: %d",
                LOCAL_PR_ARG, __entry->queue, __entry->reason,
                __entry->refcount
        )
);

TRACE_EVENT(drv_can_neg_ttlm,
            TP_PROTO(struct ieee80211_local *local,
                     struct ieee80211_sub_if_data *sdata,
                     struct ieee80211_neg_ttlm *neg_ttlm),

        TP_ARGS(local, sdata, neg_ttlm),

        TP_STRUCT__entry(LOCAL_ENTRY
                         VIF_ENTRY
                         __array(u16, downlink, sizeof(u16) * 8)
                         __array(u16, uplink, sizeof(u16) * 8)
        ),

        TP_fast_assign(LOCAL_ASSIGN;
                       VIF_ASSIGN;
                       memcpy(__entry->downlink, neg_ttlm->downlink,
                              sizeof(neg_ttlm->downlink));
                       memcpy(__entry->uplink, neg_ttlm->uplink,
                              sizeof(neg_ttlm->uplink));
        ),

        TP_printk(LOCAL_PR_FMT ", " VIF_PR_FMT, LOCAL_PR_ARG, VIF_PR_ARG)
);

TRACE_EVENT(drv_neg_ttlm_res,
            TP_PROTO(struct ieee80211_local *local,
                     struct ieee80211_sub_if_data *sdata,
                     enum ieee80211_neg_ttlm_res res,
                     struct ieee80211_neg_ttlm *neg_ttlm),

        TP_ARGS(local, sdata, res, neg_ttlm),

        TP_STRUCT__entry(LOCAL_ENTRY
                         VIF_ENTRY
                         __field(u32, res)
                         __array(u16, downlink, sizeof(u16) * 8)
                         __array(u16, uplink, sizeof(u16) * 8)
        ),

        TP_fast_assign(LOCAL_ASSIGN;
                       VIF_ASSIGN;
                       __entry->res = res;
                       memcpy(__entry->downlink, neg_ttlm->downlink,
                              sizeof(neg_ttlm->downlink));
                       memcpy(__entry->uplink, neg_ttlm->uplink,
                              sizeof(neg_ttlm->uplink));
        ),

        TP_printk(LOCAL_PR_FMT  VIF_PR_FMT " response: %d\n ",
                  LOCAL_PR_ARG, VIF_PR_ARG, __entry->res
        )
);

TRACE_EVENT(drv_prep_add_interface,
            TP_PROTO(struct ieee80211_local *local,
                     enum nl80211_iftype type),

        TP_ARGS(local, type),
        TP_STRUCT__entry(LOCAL_ENTRY
                         __field(u32, type)
        ),

        TP_fast_assign(LOCAL_ASSIGN;
                       __entry->type = type;
        ),

        TP_printk(LOCAL_PR_FMT  " type: %u\n ",
                  LOCAL_PR_ARG, __entry->type
        )
);

#endif /* !__MAC80211_DRIVER_TRACE || TRACE_HEADER_MULTI_READ */

#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH .
#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_FILE trace
#include <trace/define_trace.h>











































































































































   38 





  110 


















  325 



  318 



  209 




  204 

   30 





   30 




  204 
   30 









   23 

   98 








  105 


  106 
  106 
  105 









  105 


  104 




   29 



   28 
   27 

   12 

    8 

   16 
























































































  542 










  542 














































  100 



























    1 


    1 

















  699 




  700 





    1 





    1 








    1 












































   22 


    7 





















































   29 





    8 

    8 



   27 









   29 









    5 









   20 


   20 




















































  665 




   31 

   31 




   16 












   17 

   11 








   11 





   31 















   31 


   22 
   16 













   31 






  701 


  495 


  442 




  412 
  342 
   31 








  701 







  702 








  699 



   13 
  670 


  161 
  677 





























  699 



  702 
  701 



  698 
   27 



  701 





























  701 






















  700 


    6 






  781 







  450 


  698 
















  700 
  700 
















  700 









    7 

































































































  380 




  206 



  206 

























   34 
















   34 















   34 








   34 






  209 


  141 

  262 


  553 












  563 




  563 




  564 


  561 

  558 
   17 


  563 







  475 


  477 

   71 

   71 
    1 


   72 





   13 


   13 













   25 





   25 



   25 



















   37 




   37 


   37 



   37 




    2 



   25 

    9 








   22 










   37 














   37 







    8 
   29 




   25 


   23 


























   26 








   37 








   38 


   38 











    7 
   31 

    1 

   32 
















  517 


  514 
  518 
  517 
  516 







  518 

















  649 





















  518 



  514 



  517 

























































































































  695 



  692 

  693 























































  284 


  720 
























































































































  280 







  279 
  280 





























  270 




  271 








  696 





















  697 


  696 




  695 











   22 





   22 















































   22 


   22 

















   22 















































































































   10 
   10 

































































































   53 





   53 



































   54 









   54 






  782 






  786 
  680 

   69 

  452 















  759 




  758 







  761 











  649 














   38 







  649 


  518 




  750 

  730 
   13 
  747 










  247 









  248 









  130 








  118 




    4 


    8 





    2 





  128 
    1 





































































































































  130 








  130 


























































































  233 














  235 



   19 


  235 

  234 
  235 








  529 










  241 













  532 
  149 



  276 
   32 
  473 








  529 



  529 
  532 


  529 



  531 














































   99 


   94 

























  100 

  100 







   99 









   99 






   99 

   99 

  100 



  100 


























   96 




   96 

   96 

   22 




   86 
   22 
   62 
   63 



   94 


























































































































































































































































































































































































   46 

   45 




   46 

   46 



   46 













  396 
  399 














































   80 





















   81 




   81 



















































   19 
   12 



    1 











   16 
   13 

    6 





















    1 
   10 




    7 
    3 
    5 










    9 











    2 




    6 
























   50 

    2 



   17 










    8 



   26 


   16 

   11 


   26 

   26 








    1 


    1 







   37 

    1 










   19 
  201 










  263 




















    4 


    2 



  216 

    3 





    1 




   43 








   32 

   14 
   15 
    3 
    3 


   18 

   18 







  234 
   52 


















  213 
    7 




  212 

    7 



  213 









  227 



  250 
  239 

  257 



   48 






   10 


   10 


   10 
   10 




   10 

   10 












   10 




   10 









  267 










    1 





    3 




    3 




    3 



    2 









  255 


    1 








  213 


   28 
   22 






  235 










    6 





  248 
    1 

  242 
   15 


  200 
   48 





























   41 








  218 








    1 






  216 



   42 









  211 





   46 


  172 





  263 




    3 




  261 





  263 




  259 









   12 






    3 




    9 
























































   12 

   12 
































    5 































   30 







    2 



   28 






   10 
   25 






   20 








    5 


    5 





   20 

   19 














   19 



   18 


   16 

   16 



    4 

   12 


   14 


































































































































































































































































































































































   43 














    7 
   23 



   15 











   50 
   50 





   50 

   50 



   30 


   18 

    2 






   48 




   50 








   20 




   30 
   30 























  146 
























  147 















   10 




   10 
   10 










   10 




















































    1 

    1 














    1 

    1 












    1 

    1 

    1 










    1 











    1 

    1 
    1 



  208 







  207 






    1 



    1 
    1 



  203 



  204 

  200 








  203 







  160 
  115 












    4 
    4 














































  356 




  354 




  171 




  246 
    4 














   37 


   12 








   37 











  356 



  323 
  326 
  325 










   20 








   20 
   10 


   10 




   10 







   20 









   20 













   20 



   20 


































  113 





    1 

    2 






    2 





    1 

















   79 



   28 

   96 
   11 

   96 
   11 



  100 

    3 
    3 



    3 


  101 
    3 



    2 



   91 


    1 


   10 


   97 


    2 


    2 


   95 
    3 

   97 
    2 

   91 
    8 

   86 
   13 



   86 
   13 

   38 






   60 



   86 
    7 






   89 
    5 

   59 




   34 



   87 



    3 























   20 










   21 








    8 






    8 

    8 






    8 

    8 




    8 




    8 
    8 



    8 



    5 
    3 



    8 








    3 



   27 




















   44 












   22 

    2 
   41 


    4 



   13 
   27 












   28 
   10 





   15 



    7 

    7 

    7 





   21 










    1 



















   13 





















    8 











    8 






























   13 






   43 
   21 
   21 




















   12 

    9 
   12 


   11 



    9 
    3 






















   29 




   13 


   15 

    1 




   12 

    3 






   84 




   10 


    8 
   66 


   44 

   30 











    1 










  387 











  385 
    1 

    1 

    1 

    1 


    1 






















































  414 








    4 



















  415 



    2 







  411 





    7 
   42 
  384 





    7 
  407 














    2 

  397 

    5 




  407 






  403 



    1 






    6 



    6 


    6 















    1 





    1 






    1 
















   10 


  399 


  401 
   12 




  399 

  375 
  268 
  398 



   10 


    2 



  403 



   20 

















































   13 






   13 
   13 











































   21 






   16 


   21 




   13 







   13 







   13 

















   13 





   13 

   13 

   13 















    1 





   33 





























































   34 












   34 
   33 






   26 
    1 


    3 


    9 
    1 


   19 


   13 
   15 

   26 
    2 

   21 
    7 





   28 

   26 
    2 

   27 
    1 

   25 



    3 



   26 






   14 



    2 









   13 

   13 

   11 







    9 



















   18 
   18 




   11 


    7 








    8 




   10 






  387 






  385 













  387 























































































 1024 



 1001 


   53 
   28 




















































































































































   29 







   29 

















   29 









   29 









































   29 


















































   29 






   29 






















   29 




   29 



































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
6195
6196
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
6284
6285
6286
6287
6288
6289
6290
6291
6292
6293
6294
6295
6296
6297
6298
6299
6300
6301
6302
6303
6304
6305
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
6380
6381
6382
6383
6384
6385
6386
6387
6388
6389
6390
6391
6392
6393
6394
6395
6396
6397
6398
6399
6400
6401
6402
6403
6404
6405
6406
6407
6408
6409
6410
6411
6412
6413
6414
6415
6416
6417
6418
6419
6420
6421
6422
6423
6424
6425
6426
6427
6428
6429
6430
6431
6432
6433
6434
6435
6436
6437
6438
6439
6440
6441
6442
6443
6444
6445
6446
6447
6448
6449
6450
6451
6452
6453
6454
6455
6456
6457
6458
6459
6460
6461
6462
6463
6464
6465
6466
6467
6468
6469
6470
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480
6481
6482
6483
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516
6517
6518
6519
6520
6521
6522
6523
6524
6525
6526
6527
6528
6529
6530
6531
6532
6533
6534
6535
6536
6537
6538
6539
6540
6541
6542
6543
6544
6545
6546
6547
6548
6549
6550
6551
6552
6553
6554
6555
6556
6557
6558
6559
6560
6561
6562
6563
6564
6565
6566
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576
6577
6578
6579
6580
6581
6582
6583
6584
6585
6586
6587
6588
6589
6590
6591
6592
6593
6594
6595
6596
6597
6598
6599
6600
6601
6602
6603
6604
6605
6606
6607
6608
6609
6610
6611
6612
6613
6614
6615
6616
6617
6618
6619
6620
6621
6622
6623
6624
6625
6626
6627
6628
6629
6630
6631
6632
6633
6634
6635
6636
6637
6638
6639
6640
6641
6642
6643
6644
6645
6646
6647
6648
6649
6650
6651
6652
6653
6654
6655
6656
6657
6658
6659
6660
6661
6662
6663
6664
6665
6666
6667
6668
6669
6670
6671
6672
6673
6674
6675
6676
6677
6678
6679
6680
6681
6682
6683
6684
6685
6686
6687
6688
6689
6690
6691
6692
6693
6694
6695
6696
6697
6698
6699
6700
6701
6702
6703
6704
6705
6706
6707
6708
6709
6710
6711
6712
6713
6714
6715
6716
6717
6718
6719
6720
6721
6722
6723
6724
6725
6726
6727
6728
6729
6730
6731
6732
6733
6734
6735
6736
6737
6738
6739
6740
6741
6742
6743
6744
6745
6746
6747
6748
6749
6750
6751
6752
6753
6754
6755
6756
6757
6758
6759
6760
6761
6762
6763
6764
6765
6766
6767
6768
6769
6770
6771
6772
6773
6774
6775
6776
6777
6778
6779
6780
6781
6782
6783
6784
6785
6786
6787
6788
6789
6790
6791
6792
6793
6794
6795
6796
6797
6798
6799
6800
6801
6802
6803
6804
6805
6806
6807
6808
6809
6810
6811
6812
6813
6814
6815
6816
6817
6818
6819
6820
6821
6822
6823
6824
6825
6826
6827
6828
6829
6830
6831
6832
6833
6834
6835
6836
6837
6838
6839
6840
6841
6842
6843
6844
6845
6846
6847
6848
6849
6850
6851
6852
6853
6854
6855
6856
6857
6858
6859
6860
6861
6862
6863
6864
6865
6866
6867
6868
6869
6870
6871
6872
6873
6874
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        Linux INET6 implementation
 *        FIB front-end.
 *
 *        Authors:
 *        Pedro Roque                <roque@di.fc.ul.pt>
 */

/*        Changes:
 *
 *        YOSHIFUJI Hideaki @USAGI
 *                reworked default router selection.
 *                - respect outgoing interface
 *                - select from (probably) reachable routers (i.e.
 *                routers in REACHABLE, STALE, DELAY or PROBE states).
 *                - always select the same router if it is (probably)
 *                reachable.  otherwise, round-robin the list.
 *        Ville Nuorvala
 *                Fixed routing subtrees.
 */

#define pr_fmt(fmt) "IPv6: " fmt

#include <linux/capability.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/types.h>
#include <linux/times.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/net.h>
#include <linux/route.h>
#include <linux/netdevice.h>
#include <linux/in6.h>
#include <linux/mroute6.h>
#include <linux/init.h>
#include <linux/if_arp.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/nsproxy.h>
#include <linux/slab.h>
#include <linux/jhash.h>
#include <linux/siphash.h>
#include <net/net_namespace.h>
#include <net/snmp.h>
#include <net/ipv6.h>
#include <net/ip6_fib.h>
#include <net/ip6_route.h>
#include <net/ndisc.h>
#include <net/addrconf.h>
#include <net/tcp.h>
#include <linux/rtnetlink.h>
#include <net/dst.h>
#include <net/dst_metadata.h>
#include <net/xfrm.h>
#include <net/netevent.h>
#include <net/netlink.h>
#include <net/rtnh.h>
#include <net/lwtunnel.h>
#include <net/ip_tunnels.h>
#include <net/l3mdev.h>
#include <net/ip.h>
#include <linux/uaccess.h>
#include <linux/btf_ids.h>

#ifdef CONFIG_SYSCTL
#include <linux/sysctl.h>
#endif

static int ip6_rt_type_to_error(u8 fib6_type);

#define CREATE_TRACE_POINTS
#include <trace/events/fib6.h>
EXPORT_TRACEPOINT_SYMBOL_GPL(fib6_table_lookup);
#undef CREATE_TRACE_POINTS

enum rt6_nud_state {
        RT6_NUD_FAIL_HARD = -3,
        RT6_NUD_FAIL_PROBE = -2,
        RT6_NUD_FAIL_DO_RR = -1,
        RT6_NUD_SUCCEED = 1
};

INDIRECT_CALLABLE_SCOPE
struct dst_entry        *ip6_dst_check(struct dst_entry *dst, u32 cookie);
static unsigned int         ip6_default_advmss(const struct dst_entry *dst);
INDIRECT_CALLABLE_SCOPE
unsigned int                ip6_mtu(const struct dst_entry *dst);
static void                ip6_negative_advice(struct sock *sk,
                                            struct dst_entry *dst);
static void                ip6_dst_destroy(struct dst_entry *);
static void                ip6_dst_ifdown(struct dst_entry *,
                                       struct net_device *dev);
static void                 ip6_dst_gc(struct dst_ops *ops);

static int                ip6_pkt_discard(struct sk_buff *skb);
static int                ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb);
static int                ip6_pkt_prohibit(struct sk_buff *skb);
static int                ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb);
static void                ip6_link_failure(struct sk_buff *skb);
static void                ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
                                           struct sk_buff *skb, u32 mtu,
                                           bool confirm_neigh);
static void                rt6_do_redirect(struct dst_entry *dst, struct sock *sk,
                                        struct sk_buff *skb);
static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,
                           int strict);
static size_t rt6_nlmsg_size(struct fib6_info *f6i);
static int rt6_fill_node(struct net *net, struct sk_buff *skb,
                         struct fib6_info *rt, struct dst_entry *dst,
                         struct in6_addr *dest, struct in6_addr *src,
                         int iif, int type, u32 portid, u32 seq,
                         unsigned int flags);
static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res,
                                           const struct in6_addr *daddr,
                                           const struct in6_addr *saddr);

#ifdef CONFIG_IPV6_ROUTE_INFO
static struct fib6_info *rt6_add_route_info(struct net *net,
                                           const struct in6_addr *prefix, int prefixlen,
                                           const struct in6_addr *gwaddr,
                                           struct net_device *dev,
                                           unsigned int pref);
static struct fib6_info *rt6_get_route_info(struct net *net,
                                           const struct in6_addr *prefix, int prefixlen,
                                           const struct in6_addr *gwaddr,
                                           struct net_device *dev);
#endif

struct uncached_list {
        spinlock_t                lock;
        struct list_head        head;
};

static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt6_uncached_list);

void rt6_uncached_list_add(struct rt6_info *rt)
{
        struct uncached_list *ul = raw_cpu_ptr(&rt6_uncached_list);

        rt->dst.rt_uncached_list = ul;

        spin_lock_bh(&ul->lock);
        list_add_tail(&rt->dst.rt_uncached, &ul->head);
        spin_unlock_bh(&ul->lock);
}

void rt6_uncached_list_del(struct rt6_info *rt)
{
        if (!list_empty(&rt->dst.rt_uncached)) {
                struct uncached_list *ul = rt->dst.rt_uncached_list;

                spin_lock_bh(&ul->lock);
                list_del_init(&rt->dst.rt_uncached);
                spin_unlock_bh(&ul->lock);
        }
}

static void rt6_uncached_list_flush_dev(struct net_device *dev)
{
        int cpu;

        for_each_possible_cpu(cpu) {
                struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu);
                struct rt6_info *rt, *safe;

                if (list_empty(&ul->head))
                        continue;

                spin_lock_bh(&ul->lock);
                list_for_each_entry_safe(rt, safe, &ul->head, dst.rt_uncached) {
                        struct inet6_dev *rt_idev = rt->rt6i_idev;
                        struct net_device *rt_dev = rt->dst.dev;
                        bool handled = false;

                        if (rt_idev && rt_idev->dev == dev) {
                                rt->rt6i_idev = in6_dev_get(blackhole_netdev);
                                in6_dev_put(rt_idev);
                                handled = true;
                        }

                        if (rt_dev == dev) {
                                rt->dst.dev = blackhole_netdev;
                                netdev_ref_replace(rt_dev, blackhole_netdev,
                                                   &rt->dst.dev_tracker,
                                                   GFP_ATOMIC);
                                handled = true;
                        }
                        if (handled)
                                list_del_init(&rt->dst.rt_uncached);
                }
                spin_unlock_bh(&ul->lock);
        }
}

static inline const void *choose_neigh_daddr(const struct in6_addr *p,
                                             struct sk_buff *skb,
                                             const void *daddr)
{
        if (!ipv6_addr_any(p))
                return (const void *) p;
        else if (skb)
                return &ipv6_hdr(skb)->daddr;
        return daddr;
}

struct neighbour *ip6_neigh_lookup(const struct in6_addr *gw,
                                   struct net_device *dev,
                                   struct sk_buff *skb,
                                   const void *daddr)
{
        struct neighbour *n;

        daddr = choose_neigh_daddr(gw, skb, daddr);
        n = __ipv6_neigh_lookup(dev, daddr);
        if (n)
                return n;

        n = neigh_create(&nd_tbl, daddr, dev);
        return IS_ERR(n) ? NULL : n;
}

static struct neighbour *ip6_dst_neigh_lookup(const struct dst_entry *dst,
                                              struct sk_buff *skb,
                                              const void *daddr)
{
        const struct rt6_info *rt = dst_rt6_info(dst);

        return ip6_neigh_lookup(rt6_nexthop(rt, &in6addr_any),
                                dst->dev, skb, daddr);
}

static void ip6_confirm_neigh(const struct dst_entry *dst, const void *daddr)
{
        const struct rt6_info *rt = dst_rt6_info(dst);
        struct net_device *dev = dst->dev;

        daddr = choose_neigh_daddr(rt6_nexthop(rt, &in6addr_any), NULL, daddr);
        if (!daddr)
                return;
        if (dev->flags & (IFF_NOARP | IFF_LOOPBACK))
                return;
        if (ipv6_addr_is_multicast((const struct in6_addr *)daddr))
                return;
        __ipv6_confirm_neigh(dev, daddr);
}

static struct dst_ops ip6_dst_ops_template = {
        .family                        =        AF_INET6,
        .gc                        =        ip6_dst_gc,
        .gc_thresh                =        1024,
        .check                        =        ip6_dst_check,
        .default_advmss                =        ip6_default_advmss,
        .mtu                        =        ip6_mtu,
        .cow_metrics                =        dst_cow_metrics_generic,
        .destroy                =        ip6_dst_destroy,
        .ifdown                        =        ip6_dst_ifdown,
        .negative_advice        =        ip6_negative_advice,
        .link_failure                =        ip6_link_failure,
        .update_pmtu                =        ip6_rt_update_pmtu,
        .redirect                =        rt6_do_redirect,
        .local_out                =        __ip6_local_out,
        .neigh_lookup                =        ip6_dst_neigh_lookup,
        .confirm_neigh                =        ip6_confirm_neigh,
};

static struct dst_ops ip6_dst_blackhole_ops = {
        .family                        = AF_INET6,
        .default_advmss                = ip6_default_advmss,
        .neigh_lookup                = ip6_dst_neigh_lookup,
        .check                        = ip6_dst_check,
        .destroy                = ip6_dst_destroy,
        .cow_metrics                = dst_cow_metrics_generic,
        .update_pmtu                = dst_blackhole_update_pmtu,
        .redirect                = dst_blackhole_redirect,
        .mtu                        = dst_blackhole_mtu,
};

static const u32 ip6_template_metrics[RTAX_MAX] = {
        [RTAX_HOPLIMIT - 1] = 0,
};

static const struct fib6_info fib6_null_entry_template = {
        .fib6_flags        = (RTF_REJECT | RTF_NONEXTHOP),
        .fib6_protocol  = RTPROT_KERNEL,
        .fib6_metric        = ~(u32)0,
        .fib6_ref        = REFCOUNT_INIT(1),
        .fib6_type        = RTN_UNREACHABLE,
        .fib6_metrics        = (struct dst_metrics *)&dst_default_metrics,
};

static const struct rt6_info ip6_null_entry_template = {
        .dst = {
                .__rcuref        = RCUREF_INIT(1),
                .__use                = 1,
                .obsolete        = DST_OBSOLETE_FORCE_CHK,
                .error                = -ENETUNREACH,
                .input                = ip6_pkt_discard,
                .output                = ip6_pkt_discard_out,
        },
        .rt6i_flags        = (RTF_REJECT | RTF_NONEXTHOP),
};

#ifdef CONFIG_IPV6_MULTIPLE_TABLES

static const struct rt6_info ip6_prohibit_entry_template = {
        .dst = {
                .__rcuref        = RCUREF_INIT(1),
                .__use                = 1,
                .obsolete        = DST_OBSOLETE_FORCE_CHK,
                .error                = -EACCES,
                .input                = ip6_pkt_prohibit,
                .output                = ip6_pkt_prohibit_out,
        },
        .rt6i_flags        = (RTF_REJECT | RTF_NONEXTHOP),
};

static const struct rt6_info ip6_blk_hole_entry_template = {
        .dst = {
                .__rcuref        = RCUREF_INIT(1),
                .__use                = 1,
                .obsolete        = DST_OBSOLETE_FORCE_CHK,
                .error                = -EINVAL,
                .input                = dst_discard,
                .output                = dst_discard_out,
        },
        .rt6i_flags        = (RTF_REJECT | RTF_NONEXTHOP),
};

#endif

static void rt6_info_init(struct rt6_info *rt)
{
        memset_after(rt, 0, dst);
}

/* allocate dst with ip6_dst_ops */
struct rt6_info *ip6_dst_alloc(struct net *net, struct net_device *dev,
                               int flags)
{
        struct rt6_info *rt = dst_alloc(&net->ipv6.ip6_dst_ops, dev,
                                        DST_OBSOLETE_FORCE_CHK, flags);

        if (rt) {
                rt6_info_init(rt);
                atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc);
        }

        return rt;
}
EXPORT_SYMBOL(ip6_dst_alloc);

static void ip6_dst_destroy(struct dst_entry *dst)
{
        struct rt6_info *rt = dst_rt6_info(dst);
        struct fib6_info *from;
        struct inet6_dev *idev;

        ip_dst_metrics_put(dst);
        rt6_uncached_list_del(rt);

        idev = rt->rt6i_idev;
        if (idev) {
                rt->rt6i_idev = NULL;
                in6_dev_put(idev);
        }

        from = unrcu_pointer(xchg(&rt->from, NULL));
        fib6_info_release(from);
}

static void ip6_dst_ifdown(struct dst_entry *dst, struct net_device *dev)
{
        struct rt6_info *rt = dst_rt6_info(dst);
        struct inet6_dev *idev = rt->rt6i_idev;
        struct fib6_info *from;

        if (idev && idev->dev != blackhole_netdev) {
                struct inet6_dev *blackhole_idev = in6_dev_get(blackhole_netdev);

                if (blackhole_idev) {
                        rt->rt6i_idev = blackhole_idev;
                        in6_dev_put(idev);
                }
        }
        from = unrcu_pointer(xchg(&rt->from, NULL));
        fib6_info_release(from);
}

static bool __rt6_check_expired(const struct rt6_info *rt)
{
        if (rt->rt6i_flags & RTF_EXPIRES)
                return time_after(jiffies, rt->dst.expires);
        else
                return false;
}

static bool rt6_check_expired(const struct rt6_info *rt)
{
        struct fib6_info *from;

        from = rcu_dereference(rt->from);

        if (rt->rt6i_flags & RTF_EXPIRES) {
                if (time_after(jiffies, rt->dst.expires))
                        return true;
        } else if (from) {
                return rt->dst.obsolete != DST_OBSOLETE_FORCE_CHK ||
                        fib6_check_expired(from);
        }
        return false;
}

static struct fib6_info *
rt6_multipath_first_sibling_rcu(const struct fib6_info *rt)
{
        struct fib6_info *iter;
        struct fib6_node *fn;

        fn = rcu_dereference(rt->fib6_node);
        if (!fn)
                goto out;
        iter = rcu_dereference(fn->leaf);
        if (!iter)
                goto out;

        while (iter) {
                if (iter->fib6_metric == rt->fib6_metric &&
                    rt6_qualify_for_ecmp(iter))
                        return iter;
                iter = rcu_dereference(iter->fib6_next);
        }

out:
        return NULL;
}

void fib6_select_path(const struct net *net, struct fib6_result *res,
                      struct flowi6 *fl6, int oif, bool have_oif_match,
                      const struct sk_buff *skb, int strict)
{
        struct fib6_info *first, *match = res->f6i;
        struct fib6_info *sibling;
        int hash;

        if (!match->nh && (!match->fib6_nsiblings || have_oif_match))
                goto out;

        if (match->nh && have_oif_match && res->nh)
                return;

        if (skb)
                IP6CB(skb)->flags |= IP6SKB_MULTIPATH;

        /* We might have already computed the hash for ICMPv6 errors. In such
         * case it will always be non-zero. Otherwise now is the time to do it.
         */
        if (!fl6->mp_hash &&
            (!match->nh || nexthop_is_multipath(match->nh)))
                fl6->mp_hash = rt6_multipath_hash(net, fl6, skb, NULL);

        if (unlikely(match->nh)) {
                nexthop_path_fib6_result(res, fl6->mp_hash);
                return;
        }

        first = rt6_multipath_first_sibling_rcu(match);
        if (!first)
                goto out;

        hash = fl6->mp_hash;
        if (hash <= atomic_read(&first->fib6_nh->fib_nh_upper_bound)) {
                if (rt6_score_route(first->fib6_nh, first->fib6_flags, oif,
                                    strict) >= 0)
                        match = first;
                goto out;
        }

        list_for_each_entry_rcu(sibling, &first->fib6_siblings,
                                fib6_siblings) {
                const struct fib6_nh *nh = sibling->fib6_nh;
                int nh_upper_bound;

                nh_upper_bound = atomic_read(&nh->fib_nh_upper_bound);
                if (hash > nh_upper_bound)
                        continue;
                if (rt6_score_route(nh, sibling->fib6_flags, oif, strict) < 0)
                        break;
                match = sibling;
                break;
        }

out:
        res->f6i = match;
        res->nh = match->fib6_nh;
}

/*
 *        Route lookup. rcu_read_lock() should be held.
 */

static bool __rt6_device_match(struct net *net, const struct fib6_nh *nh,
                               const struct in6_addr *saddr, int oif, int flags)
{
        const struct net_device *dev;

        if (nh->fib_nh_flags & RTNH_F_DEAD)
                return false;

        dev = nh->fib_nh_dev;
        if (oif) {
                if (dev->ifindex == oif)
                        return true;
        } else {
                if (ipv6_chk_addr(net, saddr, dev,
                                  flags & RT6_LOOKUP_F_IFACE))
                        return true;
        }

        return false;
}

struct fib6_nh_dm_arg {
        struct net                *net;
        const struct in6_addr        *saddr;
        int                        oif;
        int                        flags;
        struct fib6_nh                *nh;
};

static int __rt6_nh_dev_match(struct fib6_nh *nh, void *_arg)
{
        struct fib6_nh_dm_arg *arg = _arg;

        arg->nh = nh;
        return __rt6_device_match(arg->net, nh, arg->saddr, arg->oif,
                                  arg->flags);
}

/* returns fib6_nh from nexthop or NULL */
static struct fib6_nh *rt6_nh_dev_match(struct net *net, struct nexthop *nh,
                                        struct fib6_result *res,
                                        const struct in6_addr *saddr,
                                        int oif, int flags)
{
        struct fib6_nh_dm_arg arg = {
                .net   = net,
                .saddr = saddr,
                .oif   = oif,
                .flags = flags,
        };

        if (nexthop_is_blackhole(nh))
                return NULL;

        if (nexthop_for_each_fib6_nh(nh, __rt6_nh_dev_match, &arg))
                return arg.nh;

        return NULL;
}

static void rt6_device_match(struct net *net, struct fib6_result *res,
                             const struct in6_addr *saddr, int oif, int flags)
{
        struct fib6_info *f6i = res->f6i;
        struct fib6_info *spf6i;
        struct fib6_nh *nh;

        if (!oif && ipv6_addr_any(saddr)) {
                if (unlikely(f6i->nh)) {
                        nh = nexthop_fib6_nh(f6i->nh);
                        if (nexthop_is_blackhole(f6i->nh))
                                goto out_blackhole;
                } else {
                        nh = f6i->fib6_nh;
                }
                if (!(nh->fib_nh_flags & RTNH_F_DEAD))
                        goto out;
        }

        for (spf6i = f6i; spf6i; spf6i = rcu_dereference(spf6i->fib6_next)) {
                bool matched = false;

                if (unlikely(spf6i->nh)) {
                        nh = rt6_nh_dev_match(net, spf6i->nh, res, saddr,
                                              oif, flags);
                        if (nh)
                                matched = true;
                } else {
                        nh = spf6i->fib6_nh;
                        if (__rt6_device_match(net, nh, saddr, oif, flags))
                                matched = true;
                }
                if (matched) {
                        res->f6i = spf6i;
                        goto out;
                }
        }

        if (oif && flags & RT6_LOOKUP_F_IFACE) {
                res->f6i = net->ipv6.fib6_null_entry;
                nh = res->f6i->fib6_nh;
                goto out;
        }

        if (unlikely(f6i->nh)) {
                nh = nexthop_fib6_nh(f6i->nh);
                if (nexthop_is_blackhole(f6i->nh))
                        goto out_blackhole;
        } else {
                nh = f6i->fib6_nh;
        }

        if (nh->fib_nh_flags & RTNH_F_DEAD) {
                res->f6i = net->ipv6.fib6_null_entry;
                nh = res->f6i->fib6_nh;
        }
out:
        res->nh = nh;
        res->fib6_type = res->f6i->fib6_type;
        res->fib6_flags = res->f6i->fib6_flags;
        return;

out_blackhole:
        res->fib6_flags |= RTF_REJECT;
        res->fib6_type = RTN_BLACKHOLE;
        res->nh = nh;
}

#ifdef CONFIG_IPV6_ROUTER_PREF
struct __rt6_probe_work {
        struct work_struct work;
        struct in6_addr target;
        struct net_device *dev;
        netdevice_tracker dev_tracker;
};

static void rt6_probe_deferred(struct work_struct *w)
{
        struct in6_addr mcaddr;
        struct __rt6_probe_work *work =
                container_of(w, struct __rt6_probe_work, work);

        addrconf_addr_solict_mult(&work->target, &mcaddr);
        ndisc_send_ns(work->dev, &work->target, &mcaddr, NULL, 0);
        netdev_put(work->dev, &work->dev_tracker);
        kfree(work);
}

static void rt6_probe(struct fib6_nh *fib6_nh)
{
        struct __rt6_probe_work *work = NULL;
        const struct in6_addr *nh_gw;
        unsigned long last_probe;
        struct neighbour *neigh;
        struct net_device *dev;
        struct inet6_dev *idev;

        /*
         * Okay, this does not seem to be appropriate
         * for now, however, we need to check if it
         * is really so; aka Router Reachability Probing.
         *
         * Router Reachability Probe MUST be rate-limited
         * to no more than one per minute.
         */
        if (!fib6_nh->fib_nh_gw_family)
                return;

        nh_gw = &fib6_nh->fib_nh_gw6;
        dev = fib6_nh->fib_nh_dev;
        rcu_read_lock();
        last_probe = READ_ONCE(fib6_nh->last_probe);
        idev = __in6_dev_get(dev);
        if (!idev)
                goto out;
        neigh = __ipv6_neigh_lookup_noref(dev, nh_gw);
        if (neigh) {
                if (READ_ONCE(neigh->nud_state) & NUD_VALID)
                        goto out;

                write_lock_bh(&neigh->lock);
                if (!(neigh->nud_state & NUD_VALID) &&
                    time_after(jiffies,
                               neigh->updated +
                               READ_ONCE(idev->cnf.rtr_probe_interval))) {
                        work = kmalloc(sizeof(*work), GFP_ATOMIC);
                        if (work)
                                __neigh_set_probe_once(neigh);
                }
                write_unlock_bh(&neigh->lock);
        } else if (time_after(jiffies, last_probe +
                                       READ_ONCE(idev->cnf.rtr_probe_interval))) {
                work = kmalloc(sizeof(*work), GFP_ATOMIC);
        }

        if (!work || cmpxchg(&fib6_nh->last_probe,
                             last_probe, jiffies) != last_probe) {
                kfree(work);
        } else {
                INIT_WORK(&work->work, rt6_probe_deferred);
                work->target = *nh_gw;
                netdev_hold(dev, &work->dev_tracker, GFP_ATOMIC);
                work->dev = dev;
                schedule_work(&work->work);
        }

out:
        rcu_read_unlock();
}
#else
static inline void rt6_probe(struct fib6_nh *fib6_nh)
{
}
#endif

/*
 * Default Router Selection (RFC 2461 6.3.6)
 */
static enum rt6_nud_state rt6_check_neigh(const struct fib6_nh *fib6_nh)
{
        enum rt6_nud_state ret = RT6_NUD_FAIL_HARD;
        struct neighbour *neigh;

        rcu_read_lock();
        neigh = __ipv6_neigh_lookup_noref(fib6_nh->fib_nh_dev,
                                          &fib6_nh->fib_nh_gw6);
        if (neigh) {
                u8 nud_state = READ_ONCE(neigh->nud_state);

                if (nud_state & NUD_VALID)
                        ret = RT6_NUD_SUCCEED;
#ifdef CONFIG_IPV6_ROUTER_PREF
                else if (!(nud_state & NUD_FAILED))
                        ret = RT6_NUD_SUCCEED;
                else
                        ret = RT6_NUD_FAIL_PROBE;
#endif
        } else {
                ret = IS_ENABLED(CONFIG_IPV6_ROUTER_PREF) ?
                      RT6_NUD_SUCCEED : RT6_NUD_FAIL_DO_RR;
        }
        rcu_read_unlock();

        return ret;
}

static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,
                           int strict)
{
        int m = 0;

        if (!oif || nh->fib_nh_dev->ifindex == oif)
                m = 2;

        if (!m && (strict & RT6_LOOKUP_F_IFACE))
                return RT6_NUD_FAIL_HARD;
#ifdef CONFIG_IPV6_ROUTER_PREF
        m |= IPV6_DECODE_PREF(IPV6_EXTRACT_PREF(fib6_flags)) << 2;
#endif
        if ((strict & RT6_LOOKUP_F_REACHABLE) &&
            !(fib6_flags & RTF_NONEXTHOP) && nh->fib_nh_gw_family) {
                int n = rt6_check_neigh(nh);
                if (n < 0)
                        return n;
        }
        return m;
}

static bool find_match(struct fib6_nh *nh, u32 fib6_flags,
                       int oif, int strict, int *mpri, bool *do_rr)
{
        bool match_do_rr = false;
        bool rc = false;
        int m;

        if (nh->fib_nh_flags & RTNH_F_DEAD)
                goto out;

        if (ip6_ignore_linkdown(nh->fib_nh_dev) &&
            nh->fib_nh_flags & RTNH_F_LINKDOWN &&
            !(strict & RT6_LOOKUP_F_IGNORE_LINKSTATE))
                goto out;

        m = rt6_score_route(nh, fib6_flags, oif, strict);
        if (m == RT6_NUD_FAIL_DO_RR) {
                match_do_rr = true;
                m = 0; /* lowest valid score */
        } else if (m == RT6_NUD_FAIL_HARD) {
                goto out;
        }

        if (strict & RT6_LOOKUP_F_REACHABLE)
                rt6_probe(nh);

        /* note that m can be RT6_NUD_FAIL_PROBE at this point */
        if (m > *mpri) {
                *do_rr = match_do_rr;
                *mpri = m;
                rc = true;
        }
out:
        return rc;
}

struct fib6_nh_frl_arg {
        u32                flags;
        int                oif;
        int                strict;
        int                *mpri;
        bool                *do_rr;
        struct fib6_nh        *nh;
};

static int rt6_nh_find_match(struct fib6_nh *nh, void *_arg)
{
        struct fib6_nh_frl_arg *arg = _arg;

        arg->nh = nh;
        return find_match(nh, arg->flags, arg->oif, arg->strict,
                          arg->mpri, arg->do_rr);
}

static void __find_rr_leaf(struct fib6_info *f6i_start,
                           struct fib6_info *nomatch, u32 metric,
                           struct fib6_result *res, struct fib6_info **cont,
                           int oif, int strict, bool *do_rr, int *mpri)
{
        struct fib6_info *f6i;

        for (f6i = f6i_start;
             f6i && f6i != nomatch;
             f6i = rcu_dereference(f6i->fib6_next)) {
                bool matched = false;
                struct fib6_nh *nh;

                if (cont && f6i->fib6_metric != metric) {
                        *cont = f6i;
                        return;
                }

                if (fib6_check_expired(f6i))
                        continue;

                if (unlikely(f6i->nh)) {
                        struct fib6_nh_frl_arg arg = {
                                .flags  = f6i->fib6_flags,
                                .oif    = oif,
                                .strict = strict,
                                .mpri   = mpri,
                                .do_rr  = do_rr
                        };

                        if (nexthop_is_blackhole(f6i->nh)) {
                                res->fib6_flags = RTF_REJECT;
                                res->fib6_type = RTN_BLACKHOLE;
                                res->f6i = f6i;
                                res->nh = nexthop_fib6_nh(f6i->nh);
                                return;
                        }
                        if (nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_find_match,
                                                     &arg)) {
                                matched = true;
                                nh = arg.nh;
                        }
                } else {
                        nh = f6i->fib6_nh;
                        if (find_match(nh, f6i->fib6_flags, oif, strict,
                                       mpri, do_rr))
                                matched = true;
                }
                if (matched) {
                        res->f6i = f6i;
                        res->nh = nh;
                        res->fib6_flags = f6i->fib6_flags;
                        res->fib6_type = f6i->fib6_type;
                }
        }
}

static void find_rr_leaf(struct fib6_node *fn, struct fib6_info *leaf,
                         struct fib6_info *rr_head, int oif, int strict,
                         bool *do_rr, struct fib6_result *res)
{
        u32 metric = rr_head->fib6_metric;
        struct fib6_info *cont = NULL;
        int mpri = -1;

        __find_rr_leaf(rr_head, NULL, metric, res, &cont,
                       oif, strict, do_rr, &mpri);

        __find_rr_leaf(leaf, rr_head, metric, res, &cont,
                       oif, strict, do_rr, &mpri);

        if (res->f6i || !cont)
                return;

        __find_rr_leaf(cont, NULL, metric, res, NULL,
                       oif, strict, do_rr, &mpri);
}

static void rt6_select(struct net *net, struct fib6_node *fn, int oif,
                       struct fib6_result *res, int strict)
{
        struct fib6_info *leaf = rcu_dereference(fn->leaf);
        struct fib6_info *rt0;
        bool do_rr = false;
        int key_plen;

        /* make sure this function or its helpers sets f6i */
        res->f6i = NULL;

        if (!leaf || leaf == net->ipv6.fib6_null_entry)
                goto out;

        rt0 = rcu_dereference(fn->rr_ptr);
        if (!rt0)
                rt0 = leaf;

        /* Double check to make sure fn is not an intermediate node
         * and fn->leaf does not points to its child's leaf
         * (This might happen if all routes under fn are deleted from
         * the tree and fib6_repair_tree() is called on the node.)
         */
        key_plen = rt0->fib6_dst.plen;
#ifdef CONFIG_IPV6_SUBTREES
        if (rt0->fib6_src.plen)
                key_plen = rt0->fib6_src.plen;
#endif
        if (fn->fn_bit != key_plen)
                goto out;

        find_rr_leaf(fn, leaf, rt0, oif, strict, &do_rr, res);
        if (do_rr) {
                struct fib6_info *next = rcu_dereference(rt0->fib6_next);

                /* no entries matched; do round-robin */
                if (!next || next->fib6_metric != rt0->fib6_metric)
                        next = leaf;

                if (next != rt0) {
                        spin_lock_bh(&leaf->fib6_table->tb6_lock);
                        /* make sure next is not being deleted from the tree */
                        if (next->fib6_node)
                                rcu_assign_pointer(fn->rr_ptr, next);
                        spin_unlock_bh(&leaf->fib6_table->tb6_lock);
                }
        }

out:
        if (!res->f6i) {
                res->f6i = net->ipv6.fib6_null_entry;
                res->nh = res->f6i->fib6_nh;
                res->fib6_flags = res->f6i->fib6_flags;
                res->fib6_type = res->f6i->fib6_type;
        }
}

static bool rt6_is_gw_or_nonexthop(const struct fib6_result *res)
{
        return (res->f6i->fib6_flags & RTF_NONEXTHOP) ||
               res->nh->fib_nh_gw_family;
}

#ifdef CONFIG_IPV6_ROUTE_INFO
int rt6_route_rcv(struct net_device *dev, u8 *opt, int len,
                  const struct in6_addr *gwaddr)
{
        struct net *net = dev_net(dev);
        struct route_info *rinfo = (struct route_info *) opt;
        struct in6_addr prefix_buf, *prefix;
        struct fib6_table *table;
        unsigned int pref;
        unsigned long lifetime;
        struct fib6_info *rt;

        if (len < sizeof(struct route_info)) {
                return -EINVAL;
        }

        /* Sanity check for prefix_len and length */
        if (rinfo->length > 3) {
                return -EINVAL;
        } else if (rinfo->prefix_len > 128) {
                return -EINVAL;
        } else if (rinfo->prefix_len > 64) {
                if (rinfo->length < 2) {
                        return -EINVAL;
                }
        } else if (rinfo->prefix_len > 0) {
                if (rinfo->length < 1) {
                        return -EINVAL;
                }
        }

        pref = rinfo->route_pref;
        if (pref == ICMPV6_ROUTER_PREF_INVALID)
                return -EINVAL;

        lifetime = addrconf_timeout_fixup(ntohl(rinfo->lifetime), HZ);

        if (rinfo->length == 3)
                prefix = (struct in6_addr *)rinfo->prefix;
        else {
                /* this function is safe */
                ipv6_addr_prefix(&prefix_buf,
                                 (struct in6_addr *)rinfo->prefix,
                                 rinfo->prefix_len);
                prefix = &prefix_buf;
        }

        if (rinfo->prefix_len == 0)
                rt = rt6_get_dflt_router(net, gwaddr, dev);
        else
                rt = rt6_get_route_info(net, prefix, rinfo->prefix_len,
                                        gwaddr, dev);

        if (rt && !lifetime) {
                ip6_del_rt(net, rt, false);
                rt = NULL;
        }

        if (!rt && lifetime)
                rt = rt6_add_route_info(net, prefix, rinfo->prefix_len, gwaddr,
                                        dev, pref);
        else if (rt)
                rt->fib6_flags = RTF_ROUTEINFO |
                                 (rt->fib6_flags & ~RTF_PREF_MASK) | RTF_PREF(pref);

        if (rt) {
                table = rt->fib6_table;
                spin_lock_bh(&table->tb6_lock);

                if (!addrconf_finite_timeout(lifetime)) {
                        fib6_clean_expires(rt);
                        fib6_remove_gc_list(rt);
                } else {
                        fib6_set_expires(rt, jiffies + HZ * lifetime);
                        fib6_add_gc_list(rt);
                }

                spin_unlock_bh(&table->tb6_lock);

                fib6_info_release(rt);
        }
        return 0;
}
#endif

/*
 *        Misc support functions
 */

/* called with rcu_lock held */
static struct net_device *ip6_rt_get_dev_rcu(const struct fib6_result *res)
{
        struct net_device *dev = res->nh->fib_nh_dev;

        if (res->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) {
                /* for copies of local routes, dst->dev needs to be the
                 * device if it is a master device, the master device if
                 * device is enslaved, and the loopback as the default
                 */
                if (netif_is_l3_slave(dev) &&
                    !rt6_need_strict(&res->f6i->fib6_dst.addr))
                        dev = l3mdev_master_dev_rcu(dev);
                else if (!netif_is_l3_master(dev))
                        dev = dev_net(dev)->loopback_dev;
                /* last case is netif_is_l3_master(dev) is true in which
                 * case we want dev returned to be dev
                 */
        }

        return dev;
}

static const int fib6_prop[RTN_MAX + 1] = {
        [RTN_UNSPEC]        = 0,
        [RTN_UNICAST]        = 0,
        [RTN_LOCAL]        = 0,
        [RTN_BROADCAST]        = 0,
        [RTN_ANYCAST]        = 0,
        [RTN_MULTICAST]        = 0,
        [RTN_BLACKHOLE]        = -EINVAL,
        [RTN_UNREACHABLE] = -EHOSTUNREACH,
        [RTN_PROHIBIT]        = -EACCES,
        [RTN_THROW]        = -EAGAIN,
        [RTN_NAT]        = -EINVAL,
        [RTN_XRESOLVE]        = -EINVAL,
};

static int ip6_rt_type_to_error(u8 fib6_type)
{
        return fib6_prop[fib6_type];
}

static unsigned short fib6_info_dst_flags(struct fib6_info *rt)
{
        unsigned short flags = 0;

        if (rt->dst_nocount)
                flags |= DST_NOCOUNT;
        if (rt->dst_nopolicy)
                flags |= DST_NOPOLICY;

        return flags;
}

static void ip6_rt_init_dst_reject(struct rt6_info *rt, u8 fib6_type)
{
        rt->dst.error = ip6_rt_type_to_error(fib6_type);

        switch (fib6_type) {
        case RTN_BLACKHOLE:
                rt->dst.output = dst_discard_out;
                rt->dst.input = dst_discard;
                break;
        case RTN_PROHIBIT:
                rt->dst.output = ip6_pkt_prohibit_out;
                rt->dst.input = ip6_pkt_prohibit;
                break;
        case RTN_THROW:
        case RTN_UNREACHABLE:
        default:
                rt->dst.output = ip6_pkt_discard_out;
                rt->dst.input = ip6_pkt_discard;
                break;
        }
}

static void ip6_rt_init_dst(struct rt6_info *rt, const struct fib6_result *res)
{
        struct fib6_info *f6i = res->f6i;

        if (res->fib6_flags & RTF_REJECT) {
                ip6_rt_init_dst_reject(rt, res->fib6_type);
                return;
        }

        rt->dst.error = 0;
        rt->dst.output = ip6_output;

        if (res->fib6_type == RTN_LOCAL || res->fib6_type == RTN_ANYCAST) {
                rt->dst.input = ip6_input;
        } else if (ipv6_addr_type(&f6i->fib6_dst.addr) & IPV6_ADDR_MULTICAST) {
                rt->dst.input = ip6_mc_input;
        } else {
                rt->dst.input = ip6_forward;
        }

        if (res->nh->fib_nh_lws) {
                rt->dst.lwtstate = lwtstate_get(res->nh->fib_nh_lws);
                lwtunnel_set_redirect(&rt->dst);
        }

        rt->dst.lastuse = jiffies;
}

/* Caller must already hold reference to @from */
static void rt6_set_from(struct rt6_info *rt, struct fib6_info *from)
{
        rt->rt6i_flags &= ~RTF_EXPIRES;
        rcu_assign_pointer(rt->from, from);
        ip_dst_init_metrics(&rt->dst, from->fib6_metrics);
}

/* Caller must already hold reference to f6i in result */
static void ip6_rt_copy_init(struct rt6_info *rt, const struct fib6_result *res)
{
        const struct fib6_nh *nh = res->nh;
        const struct net_device *dev = nh->fib_nh_dev;
        struct fib6_info *f6i = res->f6i;

        ip6_rt_init_dst(rt, res);

        rt->rt6i_dst = f6i->fib6_dst;
        rt->rt6i_idev = dev ? in6_dev_get(dev) : NULL;
        rt->rt6i_flags = res->fib6_flags;
        if (nh->fib_nh_gw_family) {
                rt->rt6i_gateway = nh->fib_nh_gw6;
                rt->rt6i_flags |= RTF_GATEWAY;
        }
        rt6_set_from(rt, f6i);
#ifdef CONFIG_IPV6_SUBTREES
        rt->rt6i_src = f6i->fib6_src;
#endif
}

static struct fib6_node* fib6_backtrack(struct fib6_node *fn,
                                        struct in6_addr *saddr)
{
        struct fib6_node *pn, *sn;
        while (1) {
                if (fn->fn_flags & RTN_TL_ROOT)
                        return NULL;
                pn = rcu_dereference(fn->parent);
                sn = FIB6_SUBTREE(pn);
                if (sn && sn != fn)
                        fn = fib6_node_lookup(sn, NULL, saddr);
                else
                        fn = pn;
                if (fn->fn_flags & RTN_RTINFO)
                        return fn;
        }
}

static bool ip6_hold_safe(struct net *net, struct rt6_info **prt)
{
        struct rt6_info *rt = *prt;

        if (dst_hold_safe(&rt->dst))
                return true;
        if (net) {
                rt = net->ipv6.ip6_null_entry;
                dst_hold(&rt->dst);
        } else {
                rt = NULL;
        }
        *prt = rt;
        return false;
}

/* called with rcu_lock held */
static struct rt6_info *ip6_create_rt_rcu(const struct fib6_result *res)
{
        struct net_device *dev = res->nh->fib_nh_dev;
        struct fib6_info *f6i = res->f6i;
        unsigned short flags;
        struct rt6_info *nrt;

        if (!fib6_info_hold_safe(f6i))
                goto fallback;

        flags = fib6_info_dst_flags(f6i);
        nrt = ip6_dst_alloc(dev_net(dev), dev, flags);
        if (!nrt) {
                fib6_info_release(f6i);
                goto fallback;
        }

        ip6_rt_copy_init(nrt, res);
        return nrt;

fallback:
        nrt = dev_net(dev)->ipv6.ip6_null_entry;
        dst_hold(&nrt->dst);
        return nrt;
}

INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_lookup(struct net *net,
                                             struct fib6_table *table,
                                             struct flowi6 *fl6,
                                             const struct sk_buff *skb,
                                             int flags)
{
        struct fib6_result res = {};
        struct fib6_node *fn;
        struct rt6_info *rt;

        rcu_read_lock();
        fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr);
restart:
        res.f6i = rcu_dereference(fn->leaf);
        if (!res.f6i)
                res.f6i = net->ipv6.fib6_null_entry;
        else
                rt6_device_match(net, &res, &fl6->saddr, fl6->flowi6_oif,
                                 flags);

        if (res.f6i == net->ipv6.fib6_null_entry) {
                fn = fib6_backtrack(fn, &fl6->saddr);
                if (fn)
                        goto restart;

                rt = net->ipv6.ip6_null_entry;
                dst_hold(&rt->dst);
                goto out;
        } else if (res.fib6_flags & RTF_REJECT) {
                goto do_create;
        }

        fib6_select_path(net, &res, fl6, fl6->flowi6_oif,
                         fl6->flowi6_oif != 0, skb, flags);

        /* Search through exception table */
        rt = rt6_find_cached_rt(&res, &fl6->daddr, &fl6->saddr);
        if (rt) {
                if (ip6_hold_safe(net, &rt))
                        dst_use_noref(&rt->dst, jiffies);
        } else {
do_create:
                rt = ip6_create_rt_rcu(&res);
        }

out:
        trace_fib6_table_lookup(net, &res, table, fl6);

        rcu_read_unlock();

        return rt;
}

struct dst_entry *ip6_route_lookup(struct net *net, struct flowi6 *fl6,
                                   const struct sk_buff *skb, int flags)
{
        return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_lookup);
}
EXPORT_SYMBOL_GPL(ip6_route_lookup);

struct rt6_info *rt6_lookup(struct net *net, const struct in6_addr *daddr,
                            const struct in6_addr *saddr, int oif,
                            const struct sk_buff *skb, int strict)
{
        struct flowi6 fl6 = {
                .flowi6_oif = oif,
                .daddr = *daddr,
        };
        struct dst_entry *dst;
        int flags = strict ? RT6_LOOKUP_F_IFACE : 0;

        if (saddr) {
                memcpy(&fl6.saddr, saddr, sizeof(*saddr));
                flags |= RT6_LOOKUP_F_HAS_SADDR;
        }

        dst = fib6_rule_lookup(net, &fl6, skb, flags, ip6_pol_route_lookup);
        if (dst->error == 0)
                return dst_rt6_info(dst);

        dst_release(dst);

        return NULL;
}
EXPORT_SYMBOL(rt6_lookup);

/* ip6_ins_rt is called with FREE table->tb6_lock.
 * It takes new route entry, the addition fails by any reason the
 * route is released.
 * Caller must hold dst before calling it.
 */

static int __ip6_ins_rt(struct fib6_info *rt, struct nl_info *info,
                        struct netlink_ext_ack *extack)
{
        int err;
        struct fib6_table *table;

        table = rt->fib6_table;
        spin_lock_bh(&table->tb6_lock);
        err = fib6_add(&table->tb6_root, rt, info, extack);
        spin_unlock_bh(&table->tb6_lock);

        return err;
}

int ip6_ins_rt(struct net *net, struct fib6_info *rt)
{
        struct nl_info info = {        .nl_net = net, };

        return __ip6_ins_rt(rt, &info, NULL);
}

static struct rt6_info *ip6_rt_cache_alloc(const struct fib6_result *res,
                                           const struct in6_addr *daddr,
                                           const struct in6_addr *saddr)
{
        struct fib6_info *f6i = res->f6i;
        struct net_device *dev;
        struct rt6_info *rt;

        /*
         *        Clone the route.
         */

        if (!fib6_info_hold_safe(f6i))
                return NULL;

        dev = ip6_rt_get_dev_rcu(res);
        rt = ip6_dst_alloc(dev_net(dev), dev, 0);
        if (!rt) {
                fib6_info_release(f6i);
                return NULL;
        }

        ip6_rt_copy_init(rt, res);
        rt->rt6i_flags |= RTF_CACHE;
        rt->rt6i_dst.addr = *daddr;
        rt->rt6i_dst.plen = 128;

        if (!rt6_is_gw_or_nonexthop(res)) {
                if (f6i->fib6_dst.plen != 128 &&
                    ipv6_addr_equal(&f6i->fib6_dst.addr, daddr))
                        rt->rt6i_flags |= RTF_ANYCAST;
#ifdef CONFIG_IPV6_SUBTREES
                if (rt->rt6i_src.plen && saddr) {
                        rt->rt6i_src.addr = *saddr;
                        rt->rt6i_src.plen = 128;
                }
#endif
        }

        return rt;
}

static struct rt6_info *ip6_rt_pcpu_alloc(const struct fib6_result *res)
{
        struct fib6_info *f6i = res->f6i;
        unsigned short flags = fib6_info_dst_flags(f6i);
        struct net_device *dev;
        struct rt6_info *pcpu_rt;

        if (!fib6_info_hold_safe(f6i))
                return NULL;

        rcu_read_lock();
        dev = ip6_rt_get_dev_rcu(res);
        pcpu_rt = ip6_dst_alloc(dev_net(dev), dev, flags | DST_NOCOUNT);
        rcu_read_unlock();
        if (!pcpu_rt) {
                fib6_info_release(f6i);
                return NULL;
        }
        ip6_rt_copy_init(pcpu_rt, res);
        pcpu_rt->rt6i_flags |= RTF_PCPU;

        if (f6i->nh)
                pcpu_rt->sernum = rt_genid_ipv6(dev_net(dev));

        return pcpu_rt;
}

static bool rt6_is_valid(const struct rt6_info *rt6)
{
        return rt6->sernum == rt_genid_ipv6(dev_net(rt6->dst.dev));
}

/* It should be called with rcu_read_lock() acquired */
static struct rt6_info *rt6_get_pcpu_route(const struct fib6_result *res)
{
        struct rt6_info *pcpu_rt;

        pcpu_rt = this_cpu_read(*res->nh->rt6i_pcpu);

        if (pcpu_rt && pcpu_rt->sernum && !rt6_is_valid(pcpu_rt)) {
                struct rt6_info *prev, **p;

                p = this_cpu_ptr(res->nh->rt6i_pcpu);
                /* Paired with READ_ONCE() in __fib6_drop_pcpu_from() */
                prev = xchg(p, NULL);
                if (prev) {
                        dst_dev_put(&prev->dst);
                        dst_release(&prev->dst);
                }

                pcpu_rt = NULL;
        }

        return pcpu_rt;
}

static struct rt6_info *rt6_make_pcpu_route(struct net *net,
                                            const struct fib6_result *res)
{
        struct rt6_info *pcpu_rt, *prev, **p;

        pcpu_rt = ip6_rt_pcpu_alloc(res);
        if (!pcpu_rt)
                return NULL;

        p = this_cpu_ptr(res->nh->rt6i_pcpu);
        prev = cmpxchg(p, NULL, pcpu_rt);
        BUG_ON(prev);

        if (res->f6i->fib6_destroying) {
                struct fib6_info *from;

                from = unrcu_pointer(xchg(&pcpu_rt->from, NULL));
                fib6_info_release(from);
        }

        return pcpu_rt;
}

/* exception hash table implementation
 */
static DEFINE_SPINLOCK(rt6_exception_lock);

/* Remove rt6_ex from hash table and free the memory
 * Caller must hold rt6_exception_lock
 */
static void rt6_remove_exception(struct rt6_exception_bucket *bucket,
                                 struct rt6_exception *rt6_ex)
{
        struct net *net;

        if (!bucket || !rt6_ex)
                return;

        net = dev_net(rt6_ex->rt6i->dst.dev);
        net->ipv6.rt6_stats->fib_rt_cache--;

        /* purge completely the exception to allow releasing the held resources:
         * some [sk] cache may keep the dst around for unlimited time
         */
        dst_dev_put(&rt6_ex->rt6i->dst);

        hlist_del_rcu(&rt6_ex->hlist);
        dst_release(&rt6_ex->rt6i->dst);
        kfree_rcu(rt6_ex, rcu);
        WARN_ON_ONCE(!bucket->depth);
        bucket->depth--;
}

/* Remove oldest rt6_ex in bucket and free the memory
 * Caller must hold rt6_exception_lock
 */
static void rt6_exception_remove_oldest(struct rt6_exception_bucket *bucket)
{
        struct rt6_exception *rt6_ex, *oldest = NULL;

        if (!bucket)
                return;

        hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) {
                if (!oldest || time_before(rt6_ex->stamp, oldest->stamp))
                        oldest = rt6_ex;
        }
        rt6_remove_exception(bucket, oldest);
}

static u32 rt6_exception_hash(const struct in6_addr *dst,
                              const struct in6_addr *src)
{
        static siphash_aligned_key_t rt6_exception_key;
        struct {
                struct in6_addr dst;
                struct in6_addr src;
        } __aligned(SIPHASH_ALIGNMENT) combined = {
                .dst = *dst,
        };
        u64 val;

        net_get_random_once(&rt6_exception_key, sizeof(rt6_exception_key));

#ifdef CONFIG_IPV6_SUBTREES
        if (src)
                combined.src = *src;
#endif
        val = siphash(&combined, sizeof(combined), &rt6_exception_key);

        return hash_64(val, FIB6_EXCEPTION_BUCKET_SIZE_SHIFT);
}

/* Helper function to find the cached rt in the hash table
 * and update bucket pointer to point to the bucket for this
 * (daddr, saddr) pair
 * Caller must hold rt6_exception_lock
 */
static struct rt6_exception *
__rt6_find_exception_spinlock(struct rt6_exception_bucket **bucket,
                              const struct in6_addr *daddr,
                              const struct in6_addr *saddr)
{
        struct rt6_exception *rt6_ex;
        u32 hval;

        if (!(*bucket) || !daddr)
                return NULL;

        hval = rt6_exception_hash(daddr, saddr);
        *bucket += hval;

        hlist_for_each_entry(rt6_ex, &(*bucket)->chain, hlist) {
                struct rt6_info *rt6 = rt6_ex->rt6i;
                bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr);

#ifdef CONFIG_IPV6_SUBTREES
                if (matched && saddr)
                        matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr);
#endif
                if (matched)
                        return rt6_ex;
        }
        return NULL;
}

/* Helper function to find the cached rt in the hash table
 * and update bucket pointer to point to the bucket for this
 * (daddr, saddr) pair
 * Caller must hold rcu_read_lock()
 */
static struct rt6_exception *
__rt6_find_exception_rcu(struct rt6_exception_bucket **bucket,
                         const struct in6_addr *daddr,
                         const struct in6_addr *saddr)
{
        struct rt6_exception *rt6_ex;
        u32 hval;

        WARN_ON_ONCE(!rcu_read_lock_held());

        if (!(*bucket) || !daddr)
                return NULL;

        hval = rt6_exception_hash(daddr, saddr);
        *bucket += hval;

        hlist_for_each_entry_rcu(rt6_ex, &(*bucket)->chain, hlist) {
                struct rt6_info *rt6 = rt6_ex->rt6i;
                bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr);

#ifdef CONFIG_IPV6_SUBTREES
                if (matched && saddr)
                        matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr);
#endif
                if (matched)
                        return rt6_ex;
        }
        return NULL;
}

static unsigned int fib6_mtu(const struct fib6_result *res)
{
        const struct fib6_nh *nh = res->nh;
        unsigned int mtu;

        if (res->f6i->fib6_pmtu) {
                mtu = res->f6i->fib6_pmtu;
        } else {
                struct net_device *dev = nh->fib_nh_dev;
                struct inet6_dev *idev;

                rcu_read_lock();
                idev = __in6_dev_get(dev);
                mtu = READ_ONCE(idev->cnf.mtu6);
                rcu_read_unlock();
        }

        mtu = min_t(unsigned int, mtu, IP6_MAX_MTU);

        return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu);
}

#define FIB6_EXCEPTION_BUCKET_FLUSHED  0x1UL

/* used when the flushed bit is not relevant, only access to the bucket
 * (ie., all bucket users except rt6_insert_exception);
 *
 * called under rcu lock; sometimes called with rt6_exception_lock held
 */
static
struct rt6_exception_bucket *fib6_nh_get_excptn_bucket(const struct fib6_nh *nh,
                                                       spinlock_t *lock)
{
        struct rt6_exception_bucket *bucket;

        if (lock)
                bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,
                                                   lockdep_is_held(lock));
        else
                bucket = rcu_dereference(nh->rt6i_exception_bucket);

        /* remove bucket flushed bit if set */
        if (bucket) {
                unsigned long p = (unsigned long)bucket;

                p &= ~FIB6_EXCEPTION_BUCKET_FLUSHED;
                bucket = (struct rt6_exception_bucket *)p;
        }

        return bucket;
}

static bool fib6_nh_excptn_bucket_flushed(struct rt6_exception_bucket *bucket)
{
        unsigned long p = (unsigned long)bucket;

        return !!(p & FIB6_EXCEPTION_BUCKET_FLUSHED);
}

/* called with rt6_exception_lock held */
static void fib6_nh_excptn_bucket_set_flushed(struct fib6_nh *nh,
                                              spinlock_t *lock)
{
        struct rt6_exception_bucket *bucket;
        unsigned long p;

        bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,
                                           lockdep_is_held(lock));

        p = (unsigned long)bucket;
        p |= FIB6_EXCEPTION_BUCKET_FLUSHED;
        bucket = (struct rt6_exception_bucket *)p;
        rcu_assign_pointer(nh->rt6i_exception_bucket, bucket);
}

static int rt6_insert_exception(struct rt6_info *nrt,
                                const struct fib6_result *res)
{
        struct net *net = dev_net(nrt->dst.dev);
        struct rt6_exception_bucket *bucket;
        struct fib6_info *f6i = res->f6i;
        struct in6_addr *src_key = NULL;
        struct rt6_exception *rt6_ex;
        struct fib6_nh *nh = res->nh;
        int max_depth;
        int err = 0;

        spin_lock_bh(&rt6_exception_lock);

        bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,
                                          lockdep_is_held(&rt6_exception_lock));
        if (!bucket) {
                bucket = kcalloc(FIB6_EXCEPTION_BUCKET_SIZE, sizeof(*bucket),
                                 GFP_ATOMIC);
                if (!bucket) {
                        err = -ENOMEM;
                        goto out;
                }
                rcu_assign_pointer(nh->rt6i_exception_bucket, bucket);
        } else if (fib6_nh_excptn_bucket_flushed(bucket)) {
                err = -EINVAL;
                goto out;
        }

#ifdef CONFIG_IPV6_SUBTREES
        /* fib6_src.plen != 0 indicates f6i is in subtree
         * and exception table is indexed by a hash of
         * both fib6_dst and fib6_src.
         * Otherwise, the exception table is indexed by
         * a hash of only fib6_dst.
         */
        if (f6i->fib6_src.plen)
                src_key = &nrt->rt6i_src.addr;
#endif
        /* rt6_mtu_change() might lower mtu on f6i.
         * Only insert this exception route if its mtu
         * is less than f6i's mtu value.
         */
        if (dst_metric_raw(&nrt->dst, RTAX_MTU) >= fib6_mtu(res)) {
                err = -EINVAL;
                goto out;
        }

        rt6_ex = __rt6_find_exception_spinlock(&bucket, &nrt->rt6i_dst.addr,
                                               src_key);
        if (rt6_ex)
                rt6_remove_exception(bucket, rt6_ex);

        rt6_ex = kzalloc(sizeof(*rt6_ex), GFP_ATOMIC);
        if (!rt6_ex) {
                err = -ENOMEM;
                goto out;
        }
        rt6_ex->rt6i = nrt;
        rt6_ex->stamp = jiffies;
        hlist_add_head_rcu(&rt6_ex->hlist, &bucket->chain);
        bucket->depth++;
        net->ipv6.rt6_stats->fib_rt_cache++;

        /* Randomize max depth to avoid some side channels attacks. */
        max_depth = FIB6_MAX_DEPTH + get_random_u32_below(FIB6_MAX_DEPTH);
        while (bucket->depth > max_depth)
                rt6_exception_remove_oldest(bucket);

out:
        spin_unlock_bh(&rt6_exception_lock);

        /* Update fn->fn_sernum to invalidate all cached dst */
        if (!err) {
                spin_lock_bh(&f6i->fib6_table->tb6_lock);
                fib6_update_sernum(net, f6i);
                fib6_add_gc_list(f6i);
                spin_unlock_bh(&f6i->fib6_table->tb6_lock);
                fib6_force_start_gc(net);
        }

        return err;
}

static void fib6_nh_flush_exceptions(struct fib6_nh *nh, struct fib6_info *from)
{
        struct rt6_exception_bucket *bucket;
        struct rt6_exception *rt6_ex;
        struct hlist_node *tmp;
        int i;

        spin_lock_bh(&rt6_exception_lock);

        bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);
        if (!bucket)
                goto out;

        /* Prevent rt6_insert_exception() to recreate the bucket list */
        if (!from)
                fib6_nh_excptn_bucket_set_flushed(nh, &rt6_exception_lock);

        for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
                hlist_for_each_entry_safe(rt6_ex, tmp, &bucket->chain, hlist) {
                        if (!from ||
                            rcu_access_pointer(rt6_ex->rt6i->from) == from)
                                rt6_remove_exception(bucket, rt6_ex);
                }
                WARN_ON_ONCE(!from && bucket->depth);
                bucket++;
        }
out:
        spin_unlock_bh(&rt6_exception_lock);
}

static int rt6_nh_flush_exceptions(struct fib6_nh *nh, void *arg)
{
        struct fib6_info *f6i = arg;

        fib6_nh_flush_exceptions(nh, f6i);

        return 0;
}

void rt6_flush_exceptions(struct fib6_info *f6i)
{
        if (f6i->nh)
                nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_flush_exceptions,
                                         f6i);
        else
                fib6_nh_flush_exceptions(f6i->fib6_nh, f6i);
}

/* Find cached rt in the hash table inside passed in rt
 * Caller has to hold rcu_read_lock()
 */
static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res,
                                           const struct in6_addr *daddr,
                                           const struct in6_addr *saddr)
{
        const struct in6_addr *src_key = NULL;
        struct rt6_exception_bucket *bucket;
        struct rt6_exception *rt6_ex;
        struct rt6_info *ret = NULL;

#ifdef CONFIG_IPV6_SUBTREES
        /* fib6i_src.plen != 0 indicates f6i is in subtree
         * and exception table is indexed by a hash of
         * both fib6_dst and fib6_src.
         * However, the src addr used to create the hash
         * might not be exactly the passed in saddr which
         * is a /128 addr from the flow.
         * So we need to use f6i->fib6_src to redo lookup
         * if the passed in saddr does not find anything.
         * (See the logic in ip6_rt_cache_alloc() on how
         * rt->rt6i_src is updated.)
         */
        if (res->f6i->fib6_src.plen)
                src_key = saddr;
find_ex:
#endif
        bucket = fib6_nh_get_excptn_bucket(res->nh, NULL);
        rt6_ex = __rt6_find_exception_rcu(&bucket, daddr, src_key);

        if (rt6_ex && !rt6_check_expired(rt6_ex->rt6i))
                ret = rt6_ex->rt6i;

#ifdef CONFIG_IPV6_SUBTREES
        /* Use fib6_src as src_key and redo lookup */
        if (!ret && src_key && src_key != &res->f6i->fib6_src.addr) {
                src_key = &res->f6i->fib6_src.addr;
                goto find_ex;
        }
#endif

        return ret;
}

/* Remove the passed in cached rt from the hash table that contains it */
static int fib6_nh_remove_exception(const struct fib6_nh *nh, int plen,
                                    const struct rt6_info *rt)
{
        const struct in6_addr *src_key = NULL;
        struct rt6_exception_bucket *bucket;
        struct rt6_exception *rt6_ex;
        int err;

        if (!rcu_access_pointer(nh->rt6i_exception_bucket))
                return -ENOENT;

        spin_lock_bh(&rt6_exception_lock);
        bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);

#ifdef CONFIG_IPV6_SUBTREES
        /* rt6i_src.plen != 0 indicates 'from' is in subtree
         * and exception table is indexed by a hash of
         * both rt6i_dst and rt6i_src.
         * Otherwise, the exception table is indexed by
         * a hash of only rt6i_dst.
         */
        if (plen)
                src_key = &rt->rt6i_src.addr;
#endif
        rt6_ex = __rt6_find_exception_spinlock(&bucket,
                                               &rt->rt6i_dst.addr,
                                               src_key);
        if (rt6_ex) {
                rt6_remove_exception(bucket, rt6_ex);
                err = 0;
        } else {
                err = -ENOENT;
        }

        spin_unlock_bh(&rt6_exception_lock);
        return err;
}

struct fib6_nh_excptn_arg {
        struct rt6_info        *rt;
        int                plen;
};

static int rt6_nh_remove_exception_rt(struct fib6_nh *nh, void *_arg)
{
        struct fib6_nh_excptn_arg *arg = _arg;
        int err;

        err = fib6_nh_remove_exception(nh, arg->plen, arg->rt);
        if (err == 0)
                return 1;

        return 0;
}

static int rt6_remove_exception_rt(struct rt6_info *rt)
{
        struct fib6_info *from;

        from = rcu_dereference(rt->from);
        if (!from || !(rt->rt6i_flags & RTF_CACHE))
                return -EINVAL;

        if (from->nh) {
                struct fib6_nh_excptn_arg arg = {
                        .rt = rt,
                        .plen = from->fib6_src.plen
                };
                int rc;

                /* rc = 1 means an entry was found */
                rc = nexthop_for_each_fib6_nh(from->nh,
                                              rt6_nh_remove_exception_rt,
                                              &arg);
                return rc ? 0 : -ENOENT;
        }

        return fib6_nh_remove_exception(from->fib6_nh,
                                        from->fib6_src.plen, rt);
}

/* Find rt6_ex which contains the passed in rt cache and
 * refresh its stamp
 */
static void fib6_nh_update_exception(const struct fib6_nh *nh, int plen,
                                     const struct rt6_info *rt)
{
        const struct in6_addr *src_key = NULL;
        struct rt6_exception_bucket *bucket;
        struct rt6_exception *rt6_ex;

        bucket = fib6_nh_get_excptn_bucket(nh, NULL);
#ifdef CONFIG_IPV6_SUBTREES
        /* rt6i_src.plen != 0 indicates 'from' is in subtree
         * and exception table is indexed by a hash of
         * both rt6i_dst and rt6i_src.
         * Otherwise, the exception table is indexed by
         * a hash of only rt6i_dst.
         */
        if (plen)
                src_key = &rt->rt6i_src.addr;
#endif
        rt6_ex = __rt6_find_exception_rcu(&bucket, &rt->rt6i_dst.addr, src_key);
        if (rt6_ex)
                rt6_ex->stamp = jiffies;
}

struct fib6_nh_match_arg {
        const struct net_device *dev;
        const struct in6_addr        *gw;
        struct fib6_nh                *match;
};

/* determine if fib6_nh has given device and gateway */
static int fib6_nh_find_match(struct fib6_nh *nh, void *_arg)
{
        struct fib6_nh_match_arg *arg = _arg;

        if (arg->dev != nh->fib_nh_dev ||
            (arg->gw && !nh->fib_nh_gw_family) ||
            (!arg->gw && nh->fib_nh_gw_family) ||
            (arg->gw && !ipv6_addr_equal(arg->gw, &nh->fib_nh_gw6)))
                return 0;

        arg->match = nh;

        /* found a match, break the loop */
        return 1;
}

static void rt6_update_exception_stamp_rt(struct rt6_info *rt)
{
        struct fib6_info *from;
        struct fib6_nh *fib6_nh;

        rcu_read_lock();

        from = rcu_dereference(rt->from);
        if (!from || !(rt->rt6i_flags & RTF_CACHE))
                goto unlock;

        if (from->nh) {
                struct fib6_nh_match_arg arg = {
                        .dev = rt->dst.dev,
                        .gw = &rt->rt6i_gateway,
                };

                nexthop_for_each_fib6_nh(from->nh, fib6_nh_find_match, &arg);

                if (!arg.match)
                        goto unlock;
                fib6_nh = arg.match;
        } else {
                fib6_nh = from->fib6_nh;
        }
        fib6_nh_update_exception(fib6_nh, from->fib6_src.plen, rt);
unlock:
        rcu_read_unlock();
}

static bool rt6_mtu_change_route_allowed(struct inet6_dev *idev,
                                         struct rt6_info *rt, int mtu)
{
        /* If the new MTU is lower than the route PMTU, this new MTU will be the
         * lowest MTU in the path: always allow updating the route PMTU to
         * reflect PMTU decreases.
         *
         * If the new MTU is higher, and the route PMTU is equal to the local
         * MTU, this means the old MTU is the lowest in the path, so allow
         * updating it: if other nodes now have lower MTUs, PMTU discovery will
         * handle this.
         */

        if (dst_mtu(&rt->dst) >= mtu)
                return true;

        if (dst_mtu(&rt->dst) == idev->cnf.mtu6)
                return true;

        return false;
}

static void rt6_exceptions_update_pmtu(struct inet6_dev *idev,
                                       const struct fib6_nh *nh, int mtu)
{
        struct rt6_exception_bucket *bucket;
        struct rt6_exception *rt6_ex;
        int i;

        bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);
        if (!bucket)
                return;

        for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
                hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) {
                        struct rt6_info *entry = rt6_ex->rt6i;

                        /* For RTF_CACHE with rt6i_pmtu == 0 (i.e. a redirected
                         * route), the metrics of its rt->from have already
                         * been updated.
                         */
                        if (dst_metric_raw(&entry->dst, RTAX_MTU) &&
                            rt6_mtu_change_route_allowed(idev, entry, mtu))
                                dst_metric_set(&entry->dst, RTAX_MTU, mtu);
                }
                bucket++;
        }
}

#define RTF_CACHE_GATEWAY        (RTF_GATEWAY | RTF_CACHE)

static void fib6_nh_exceptions_clean_tohost(const struct fib6_nh *nh,
                                            const struct in6_addr *gateway)
{
        struct rt6_exception_bucket *bucket;
        struct rt6_exception *rt6_ex;
        struct hlist_node *tmp;
        int i;

        if (!rcu_access_pointer(nh->rt6i_exception_bucket))
                return;

        spin_lock_bh(&rt6_exception_lock);
        bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);
        if (bucket) {
                for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
                        hlist_for_each_entry_safe(rt6_ex, tmp,
                                                  &bucket->chain, hlist) {
                                struct rt6_info *entry = rt6_ex->rt6i;

                                if ((entry->rt6i_flags & RTF_CACHE_GATEWAY) ==
                                    RTF_CACHE_GATEWAY &&
                                    ipv6_addr_equal(gateway,
                                                    &entry->rt6i_gateway)) {
                                        rt6_remove_exception(bucket, rt6_ex);
                                }
                        }
                        bucket++;
                }
        }

        spin_unlock_bh(&rt6_exception_lock);
}

static void rt6_age_examine_exception(struct rt6_exception_bucket *bucket,
                                      struct rt6_exception *rt6_ex,
                                      struct fib6_gc_args *gc_args,
                                      unsigned long now)
{
        struct rt6_info *rt = rt6_ex->rt6i;

        /* we are pruning and obsoleting aged-out and non gateway exceptions
         * even if others have still references to them, so that on next
         * dst_check() such references can be dropped.
         * EXPIRES exceptions - e.g. pmtu-generated ones are pruned when
         * expired, independently from their aging, as per RFC 8201 section 4
         */
        if (!(rt->rt6i_flags & RTF_EXPIRES)) {
                if (time_after_eq(now, rt->dst.lastuse + gc_args->timeout)) {
                        pr_debug("aging clone %p\n", rt);
                        rt6_remove_exception(bucket, rt6_ex);
                        return;
                }
        } else if (time_after(jiffies, rt->dst.expires)) {
                pr_debug("purging expired route %p\n", rt);
                rt6_remove_exception(bucket, rt6_ex);
                return;
        }

        if (rt->rt6i_flags & RTF_GATEWAY) {
                struct neighbour *neigh;

                neigh = __ipv6_neigh_lookup_noref(rt->dst.dev, &rt->rt6i_gateway);

                if (!(neigh && (neigh->flags & NTF_ROUTER))) {
                        pr_debug("purging route %p via non-router but gateway\n",
                                 rt);
                        rt6_remove_exception(bucket, rt6_ex);
                        return;
                }
        }

        gc_args->more++;
}

static void fib6_nh_age_exceptions(const struct fib6_nh *nh,
                                   struct fib6_gc_args *gc_args,
                                   unsigned long now)
{
        struct rt6_exception_bucket *bucket;
        struct rt6_exception *rt6_ex;
        struct hlist_node *tmp;
        int i;

        if (!rcu_access_pointer(nh->rt6i_exception_bucket))
                return;

        rcu_read_lock_bh();
        spin_lock(&rt6_exception_lock);
        bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);
        if (bucket) {
                for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
                        hlist_for_each_entry_safe(rt6_ex, tmp,
                                                  &bucket->chain, hlist) {
                                rt6_age_examine_exception(bucket, rt6_ex,
                                                          gc_args, now);
                        }
                        bucket++;
                }
        }
        spin_unlock(&rt6_exception_lock);
        rcu_read_unlock_bh();
}

struct fib6_nh_age_excptn_arg {
        struct fib6_gc_args        *gc_args;
        unsigned long                now;
};

static int rt6_nh_age_exceptions(struct fib6_nh *nh, void *_arg)
{
        struct fib6_nh_age_excptn_arg *arg = _arg;

        fib6_nh_age_exceptions(nh, arg->gc_args, arg->now);
        return 0;
}

void rt6_age_exceptions(struct fib6_info *f6i,
                        struct fib6_gc_args *gc_args,
                        unsigned long now)
{
        if (f6i->nh) {
                struct fib6_nh_age_excptn_arg arg = {
                        .gc_args = gc_args,
                        .now = now
                };

                nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_age_exceptions,
                                         &arg);
        } else {
                fib6_nh_age_exceptions(f6i->fib6_nh, gc_args, now);
        }
}

/* must be called with rcu lock held */
int fib6_table_lookup(struct net *net, struct fib6_table *table, int oif,
                      struct flowi6 *fl6, struct fib6_result *res, int strict)
{
        struct fib6_node *fn, *saved_fn;

        fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr);
        saved_fn = fn;

redo_rt6_select:
        rt6_select(net, fn, oif, res, strict);
        if (res->f6i == net->ipv6.fib6_null_entry) {
                fn = fib6_backtrack(fn, &fl6->saddr);
                if (fn)
                        goto redo_rt6_select;
                else if (strict & RT6_LOOKUP_F_REACHABLE) {
                        /* also consider unreachable route */
                        strict &= ~RT6_LOOKUP_F_REACHABLE;
                        fn = saved_fn;
                        goto redo_rt6_select;
                }
        }

        trace_fib6_table_lookup(net, res, table, fl6);

        return 0;
}

struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table,
                               int oif, struct flowi6 *fl6,
                               const struct sk_buff *skb, int flags)
{
        struct fib6_result res = {};
        struct rt6_info *rt = NULL;
        int strict = 0;

        WARN_ON_ONCE((flags & RT6_LOOKUP_F_DST_NOREF) &&
                     !rcu_read_lock_held());

        strict |= flags & RT6_LOOKUP_F_IFACE;
        strict |= flags & RT6_LOOKUP_F_IGNORE_LINKSTATE;
        if (READ_ONCE(net->ipv6.devconf_all->forwarding) == 0)
                strict |= RT6_LOOKUP_F_REACHABLE;

        rcu_read_lock();

        fib6_table_lookup(net, table, oif, fl6, &res, strict);
        if (res.f6i == net->ipv6.fib6_null_entry)
                goto out;

        fib6_select_path(net, &res, fl6, oif, false, skb, strict);

        /*Search through exception table */
        rt = rt6_find_cached_rt(&res, &fl6->daddr, &fl6->saddr);
        if (rt) {
                goto out;
        } else if (unlikely((fl6->flowi6_flags & FLOWI_FLAG_KNOWN_NH) &&
                            !res.nh->fib_nh_gw_family)) {
                /* Create a RTF_CACHE clone which will not be
                 * owned by the fib6 tree.  It is for the special case where
                 * the daddr in the skb during the neighbor look-up is different
                 * from the fl6->daddr used to look-up route here.
                 */
                rt = ip6_rt_cache_alloc(&res, &fl6->daddr, NULL);

                if (rt) {
                        /* 1 refcnt is taken during ip6_rt_cache_alloc().
                         * As rt6_uncached_list_add() does not consume refcnt,
                         * this refcnt is always returned to the caller even
                         * if caller sets RT6_LOOKUP_F_DST_NOREF flag.
                         */
                        rt6_uncached_list_add(rt);
                        rcu_read_unlock();

                        return rt;
                }
        } else {
                /* Get a percpu copy */
                local_bh_disable();
                rt = rt6_get_pcpu_route(&res);

                if (!rt)
                        rt = rt6_make_pcpu_route(net, &res);

                local_bh_enable();
        }
out:
        if (!rt)
                rt = net->ipv6.ip6_null_entry;
        if (!(flags & RT6_LOOKUP_F_DST_NOREF))
                ip6_hold_safe(net, &rt);
        rcu_read_unlock();

        return rt;
}
EXPORT_SYMBOL_GPL(ip6_pol_route);

INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_input(struct net *net,
                                            struct fib6_table *table,
                                            struct flowi6 *fl6,
                                            const struct sk_buff *skb,
                                            int flags)
{
        return ip6_pol_route(net, table, fl6->flowi6_iif, fl6, skb, flags);
}

struct dst_entry *ip6_route_input_lookup(struct net *net,
                                         struct net_device *dev,
                                         struct flowi6 *fl6,
                                         const struct sk_buff *skb,
                                         int flags)
{
        if (rt6_need_strict(&fl6->daddr) && dev->type != ARPHRD_PIMREG)
                flags |= RT6_LOOKUP_F_IFACE;

        return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_input);
}
EXPORT_SYMBOL_GPL(ip6_route_input_lookup);

static void ip6_multipath_l3_keys(const struct sk_buff *skb,
                                  struct flow_keys *keys,
                                  struct flow_keys *flkeys)
{
        const struct ipv6hdr *outer_iph = ipv6_hdr(skb);
        const struct ipv6hdr *key_iph = outer_iph;
        struct flow_keys *_flkeys = flkeys;
        const struct ipv6hdr *inner_iph;
        const struct icmp6hdr *icmph;
        struct ipv6hdr _inner_iph;
        struct icmp6hdr _icmph;

        if (likely(outer_iph->nexthdr != IPPROTO_ICMPV6))
                goto out;

        icmph = skb_header_pointer(skb, skb_transport_offset(skb),
                                   sizeof(_icmph), &_icmph);
        if (!icmph)
                goto out;

        if (!icmpv6_is_err(icmph->icmp6_type))
                goto out;

        inner_iph = skb_header_pointer(skb,
                                       skb_transport_offset(skb) + sizeof(*icmph),
                                       sizeof(_inner_iph), &_inner_iph);
        if (!inner_iph)
                goto out;

        key_iph = inner_iph;
        _flkeys = NULL;
out:
        if (_flkeys) {
                keys->addrs.v6addrs.src = _flkeys->addrs.v6addrs.src;
                keys->addrs.v6addrs.dst = _flkeys->addrs.v6addrs.dst;
                keys->tags.flow_label = _flkeys->tags.flow_label;
                keys->basic.ip_proto = _flkeys->basic.ip_proto;
        } else {
                keys->addrs.v6addrs.src = key_iph->saddr;
                keys->addrs.v6addrs.dst = key_iph->daddr;
                keys->tags.flow_label = ip6_flowlabel(key_iph);
                keys->basic.ip_proto = key_iph->nexthdr;
        }
}

static u32 rt6_multipath_custom_hash_outer(const struct net *net,
                                           const struct sk_buff *skb,
                                           bool *p_has_inner)
{
        u32 hash_fields = ip6_multipath_hash_fields(net);
        struct flow_keys keys, hash_keys;

        if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK))
                return 0;

        memset(&hash_keys, 0, sizeof(hash_keys));
        skb_flow_dissect_flow_keys(skb, &keys, FLOW_DISSECTOR_F_STOP_AT_ENCAP);

        hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
        if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP)
                hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src;
        if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP)
                hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst;
        if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
                hash_keys.basic.ip_proto = keys.basic.ip_proto;
        if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL)
                hash_keys.tags.flow_label = keys.tags.flow_label;
        if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT)
                hash_keys.ports.src = keys.ports.src;
        if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
                hash_keys.ports.dst = keys.ports.dst;

        *p_has_inner = !!(keys.control.flags & FLOW_DIS_ENCAPSULATION);
        return fib_multipath_hash_from_keys(net, &hash_keys);
}

static u32 rt6_multipath_custom_hash_inner(const struct net *net,
                                           const struct sk_buff *skb,
                                           bool has_inner)
{
        u32 hash_fields = ip6_multipath_hash_fields(net);
        struct flow_keys keys, hash_keys;

        /* We assume the packet carries an encapsulation, but if none was
         * encountered during dissection of the outer flow, then there is no
         * point in calling the flow dissector again.
         */
        if (!has_inner)
                return 0;

        if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK))
                return 0;

        memset(&hash_keys, 0, sizeof(hash_keys));
        skb_flow_dissect_flow_keys(skb, &keys, 0);

        if (!(keys.control.flags & FLOW_DIS_ENCAPSULATION))
                return 0;

        if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
                hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
                if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP)
                        hash_keys.addrs.v4addrs.src = keys.addrs.v4addrs.src;
                if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP)
                        hash_keys.addrs.v4addrs.dst = keys.addrs.v4addrs.dst;
        } else if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
                hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
                if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP)
                        hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src;
                if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP)
                        hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst;
                if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL)
                        hash_keys.tags.flow_label = keys.tags.flow_label;
        }

        if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO)
                hash_keys.basic.ip_proto = keys.basic.ip_proto;
        if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT)
                hash_keys.ports.src = keys.ports.src;
        if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT)
                hash_keys.ports.dst = keys.ports.dst;

        return fib_multipath_hash_from_keys(net, &hash_keys);
}

static u32 rt6_multipath_custom_hash_skb(const struct net *net,
                                         const struct sk_buff *skb)
{
        u32 mhash, mhash_inner;
        bool has_inner = true;

        mhash = rt6_multipath_custom_hash_outer(net, skb, &has_inner);
        mhash_inner = rt6_multipath_custom_hash_inner(net, skb, has_inner);

        return jhash_2words(mhash, mhash_inner, 0);
}

static u32 rt6_multipath_custom_hash_fl6(const struct net *net,
                                         const struct flowi6 *fl6)
{
        u32 hash_fields = ip6_multipath_hash_fields(net);
        struct flow_keys hash_keys;

        if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK))
                return 0;

        memset(&hash_keys, 0, sizeof(hash_keys));
        hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
        if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP)
                hash_keys.addrs.v6addrs.src = fl6->saddr;
        if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP)
                hash_keys.addrs.v6addrs.dst = fl6->daddr;
        if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
                hash_keys.basic.ip_proto = fl6->flowi6_proto;
        if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL)
                hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6);
        if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT)
                hash_keys.ports.src = fl6->fl6_sport;
        if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
                hash_keys.ports.dst = fl6->fl6_dport;

        return fib_multipath_hash_from_keys(net, &hash_keys);
}

/* if skb is set it will be used and fl6 can be NULL */
u32 rt6_multipath_hash(const struct net *net, const struct flowi6 *fl6,
                       const struct sk_buff *skb, struct flow_keys *flkeys)
{
        struct flow_keys hash_keys;
        u32 mhash = 0;

        switch (ip6_multipath_hash_policy(net)) {
        case 0:
                memset(&hash_keys, 0, sizeof(hash_keys));
                hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
                if (skb) {
                        ip6_multipath_l3_keys(skb, &hash_keys, flkeys);
                } else {
                        hash_keys.addrs.v6addrs.src = fl6->saddr;
                        hash_keys.addrs.v6addrs.dst = fl6->daddr;
                        hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6);
                        hash_keys.basic.ip_proto = fl6->flowi6_proto;
                }
                mhash = fib_multipath_hash_from_keys(net, &hash_keys);
                break;
        case 1:
                if (skb) {
                        unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP;
                        struct flow_keys keys;

                        /* short-circuit if we already have L4 hash present */
                        if (skb->l4_hash)
                                return skb_get_hash_raw(skb) >> 1;

                        memset(&hash_keys, 0, sizeof(hash_keys));

                        if (!flkeys) {
                                skb_flow_dissect_flow_keys(skb, &keys, flag);
                                flkeys = &keys;
                        }
                        hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
                        hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src;
                        hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst;
                        hash_keys.ports.src = flkeys->ports.src;
                        hash_keys.ports.dst = flkeys->ports.dst;
                        hash_keys.basic.ip_proto = flkeys->basic.ip_proto;
                } else {
                        memset(&hash_keys, 0, sizeof(hash_keys));
                        hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
                        hash_keys.addrs.v6addrs.src = fl6->saddr;
                        hash_keys.addrs.v6addrs.dst = fl6->daddr;
                        hash_keys.ports.src = fl6->fl6_sport;
                        hash_keys.ports.dst = fl6->fl6_dport;
                        hash_keys.basic.ip_proto = fl6->flowi6_proto;
                }
                mhash = fib_multipath_hash_from_keys(net, &hash_keys);
                break;
        case 2:
                memset(&hash_keys, 0, sizeof(hash_keys));
                hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
                if (skb) {
                        struct flow_keys keys;

                        if (!flkeys) {
                                skb_flow_dissect_flow_keys(skb, &keys, 0);
                                flkeys = &keys;
                        }

                        /* Inner can be v4 or v6 */
                        if (flkeys->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
                                hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
                                hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src;
                                hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst;
                        } else if (flkeys->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
                                hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
                                hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src;
                                hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst;
                                hash_keys.tags.flow_label = flkeys->tags.flow_label;
                                hash_keys.basic.ip_proto = flkeys->basic.ip_proto;
                        } else {
                                /* Same as case 0 */
                                hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
                                ip6_multipath_l3_keys(skb, &hash_keys, flkeys);
                        }
                } else {
                        /* Same as case 0 */
                        hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
                        hash_keys.addrs.v6addrs.src = fl6->saddr;
                        hash_keys.addrs.v6addrs.dst = fl6->daddr;
                        hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6);
                        hash_keys.basic.ip_proto = fl6->flowi6_proto;
                }
                mhash = fib_multipath_hash_from_keys(net, &hash_keys);
                break;
        case 3:
                if (skb)
                        mhash = rt6_multipath_custom_hash_skb(net, skb);
                else
                        mhash = rt6_multipath_custom_hash_fl6(net, fl6);
                break;
        }

        return mhash >> 1;
}

/* Called with rcu held */
void ip6_route_input(struct sk_buff *skb)
{
        const struct ipv6hdr *iph = ipv6_hdr(skb);
        struct net *net = dev_net(skb->dev);
        int flags = RT6_LOOKUP_F_HAS_SADDR | RT6_LOOKUP_F_DST_NOREF;
        struct ip_tunnel_info *tun_info;
        struct flowi6 fl6 = {
                .flowi6_iif = skb->dev->ifindex,
                .daddr = iph->daddr,
                .saddr = iph->saddr,
                .flowlabel = ip6_flowinfo(iph),
                .flowi6_mark = skb->mark,
                .flowi6_proto = iph->nexthdr,
        };
        struct flow_keys *flkeys = NULL, _flkeys;

        tun_info = skb_tunnel_info(skb);
        if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
                fl6.flowi6_tun_key.tun_id = tun_info->key.tun_id;

        if (fib6_rules_early_flow_dissect(net, skb, &fl6, &_flkeys))
                flkeys = &_flkeys;

        if (unlikely(fl6.flowi6_proto == IPPROTO_ICMPV6))
                fl6.mp_hash = rt6_multipath_hash(net, &fl6, skb, flkeys);
        skb_dst_drop(skb);
        skb_dst_set_noref(skb, ip6_route_input_lookup(net, skb->dev,
                                                      &fl6, skb, flags));
}

INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_output(struct net *net,
                                             struct fib6_table *table,
                                             struct flowi6 *fl6,
                                             const struct sk_buff *skb,
                                             int flags)
{
        return ip6_pol_route(net, table, fl6->flowi6_oif, fl6, skb, flags);
}

static struct dst_entry *ip6_route_output_flags_noref(struct net *net,
                                                      const struct sock *sk,
                                                      struct flowi6 *fl6,
                                                      int flags)
{
        bool any_src;

        if (ipv6_addr_type(&fl6->daddr) &
            (IPV6_ADDR_MULTICAST | IPV6_ADDR_LINKLOCAL)) {
                struct dst_entry *dst;

                /* This function does not take refcnt on the dst */
                dst = l3mdev_link_scope_lookup(net, fl6);
                if (dst)
                        return dst;
        }

        fl6->flowi6_iif = LOOPBACK_IFINDEX;

        flags |= RT6_LOOKUP_F_DST_NOREF;
        any_src = ipv6_addr_any(&fl6->saddr);
        if ((sk && sk->sk_bound_dev_if) || rt6_need_strict(&fl6->daddr) ||
            (fl6->flowi6_oif && any_src))
                flags |= RT6_LOOKUP_F_IFACE;

        if (!any_src)
                flags |= RT6_LOOKUP_F_HAS_SADDR;
        else if (sk)
                flags |= rt6_srcprefs2flags(READ_ONCE(inet6_sk(sk)->srcprefs));

        return fib6_rule_lookup(net, fl6, NULL, flags, ip6_pol_route_output);
}

struct dst_entry *ip6_route_output_flags(struct net *net,
                                         const struct sock *sk,
                                         struct flowi6 *fl6,
                                         int flags)
{
        struct dst_entry *dst;
        struct rt6_info *rt6;

        rcu_read_lock();
        dst = ip6_route_output_flags_noref(net, sk, fl6, flags);
        rt6 = dst_rt6_info(dst);
        /* For dst cached in uncached_list, refcnt is already taken. */
        if (list_empty(&rt6->dst.rt_uncached) && !dst_hold_safe(dst)) {
                dst = &net->ipv6.ip6_null_entry->dst;
                dst_hold(dst);
        }
        rcu_read_unlock();

        return dst;
}
EXPORT_SYMBOL_GPL(ip6_route_output_flags);

struct dst_entry *ip6_blackhole_route(struct net *net, struct dst_entry *dst_orig)
{
        struct rt6_info *rt, *ort = dst_rt6_info(dst_orig);
        struct net_device *loopback_dev = net->loopback_dev;
        struct dst_entry *new = NULL;

        rt = dst_alloc(&ip6_dst_blackhole_ops, loopback_dev,
                       DST_OBSOLETE_DEAD, 0);
        if (rt) {
                rt6_info_init(rt);
                atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc);

                new = &rt->dst;
                new->__use = 1;
                new->input = dst_discard;
                new->output = dst_discard_out;

                dst_copy_metrics(new, &ort->dst);

                rt->rt6i_idev = in6_dev_get(loopback_dev);
                rt->rt6i_gateway = ort->rt6i_gateway;
                rt->rt6i_flags = ort->rt6i_flags & ~RTF_PCPU;

                memcpy(&rt->rt6i_dst, &ort->rt6i_dst, sizeof(struct rt6key));
#ifdef CONFIG_IPV6_SUBTREES
                memcpy(&rt->rt6i_src, &ort->rt6i_src, sizeof(struct rt6key));
#endif
        }

        dst_release(dst_orig);
        return new ? new : ERR_PTR(-ENOMEM);
}

/*
 *        Destination cache support functions
 */

static bool fib6_check(struct fib6_info *f6i, u32 cookie)
{
        u32 rt_cookie = 0;

        if (!fib6_get_cookie_safe(f6i, &rt_cookie) || rt_cookie != cookie)
                return false;

        if (fib6_check_expired(f6i))
                return false;

        return true;
}

static struct dst_entry *rt6_check(struct rt6_info *rt,
                                   struct fib6_info *from,
                                   u32 cookie)
{
        u32 rt_cookie = 0;

        if (!from || !fib6_get_cookie_safe(from, &rt_cookie) ||
            rt_cookie != cookie)
                return NULL;

        if (rt6_check_expired(rt))
                return NULL;

        return &rt->dst;
}

static struct dst_entry *rt6_dst_from_check(struct rt6_info *rt,
                                            struct fib6_info *from,
                                            u32 cookie)
{
        if (!__rt6_check_expired(rt) &&
            rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
            fib6_check(from, cookie))
                return &rt->dst;
        else
                return NULL;
}

INDIRECT_CALLABLE_SCOPE struct dst_entry *ip6_dst_check(struct dst_entry *dst,
                                                        u32 cookie)
{
        struct dst_entry *dst_ret;
        struct fib6_info *from;
        struct rt6_info *rt;

        rt = dst_rt6_info(dst);

        if (rt->sernum)
                return rt6_is_valid(rt) ? dst : NULL;

        rcu_read_lock();

        /* All IPV6 dsts are created with ->obsolete set to the value
         * DST_OBSOLETE_FORCE_CHK which forces validation calls down
         * into this function always.
         */

        from = rcu_dereference(rt->from);

        if (from && (rt->rt6i_flags & RTF_PCPU ||
            unlikely(!list_empty(&rt->dst.rt_uncached))))
                dst_ret = rt6_dst_from_check(rt, from, cookie);
        else
                dst_ret = rt6_check(rt, from, cookie);

        rcu_read_unlock();

        return dst_ret;
}
EXPORT_INDIRECT_CALLABLE(ip6_dst_check);

static void ip6_negative_advice(struct sock *sk,
                                struct dst_entry *dst)
{
        struct rt6_info *rt = dst_rt6_info(dst);

        if (rt->rt6i_flags & RTF_CACHE) {
                rcu_read_lock();
                if (rt6_check_expired(rt)) {
                        /* rt/dst can not be destroyed yet,
                         * because of rcu_read_lock()
                         */
                        sk_dst_reset(sk);
                        rt6_remove_exception_rt(rt);
                }
                rcu_read_unlock();
                return;
        }
        sk_dst_reset(sk);
}

static void ip6_link_failure(struct sk_buff *skb)
{
        struct rt6_info *rt;

        icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0);

        rt = dst_rt6_info(skb_dst(skb));
        if (rt) {
                rcu_read_lock();
                if (rt->rt6i_flags & RTF_CACHE) {
                        rt6_remove_exception_rt(rt);
                } else {
                        struct fib6_info *from;
                        struct fib6_node *fn;

                        from = rcu_dereference(rt->from);
                        if (from) {
                                fn = rcu_dereference(from->fib6_node);
                                if (fn && (rt->rt6i_flags & RTF_DEFAULT))
                                        WRITE_ONCE(fn->fn_sernum, -1);
                        }
                }
                rcu_read_unlock();
        }
}

static void rt6_update_expires(struct rt6_info *rt0, int timeout)
{
        if (!(rt0->rt6i_flags & RTF_EXPIRES)) {
                struct fib6_info *from;

                rcu_read_lock();
                from = rcu_dereference(rt0->from);
                if (from)
                        rt0->dst.expires = from->expires;
                rcu_read_unlock();
        }

        dst_set_expires(&rt0->dst, timeout);
        rt0->rt6i_flags |= RTF_EXPIRES;
}

static void rt6_do_update_pmtu(struct rt6_info *rt, u32 mtu)
{
        struct net *net = dev_net(rt->dst.dev);

        dst_metric_set(&rt->dst, RTAX_MTU, mtu);
        rt->rt6i_flags |= RTF_MODIFIED;
        rt6_update_expires(rt, net->ipv6.sysctl.ip6_rt_mtu_expires);
}

static bool rt6_cache_allowed_for_pmtu(const struct rt6_info *rt)
{
        return !(rt->rt6i_flags & RTF_CACHE) &&
                (rt->rt6i_flags & RTF_PCPU || rcu_access_pointer(rt->from));
}

static void __ip6_rt_update_pmtu(struct dst_entry *dst, const struct sock *sk,
                                 const struct ipv6hdr *iph, u32 mtu,
                                 bool confirm_neigh)
{
        const struct in6_addr *daddr, *saddr;
        struct rt6_info *rt6 = dst_rt6_info(dst);

        /* Note: do *NOT* check dst_metric_locked(dst, RTAX_MTU)
         * IPv6 pmtu discovery isn't optional, so 'mtu lock' cannot disable it.
         * [see also comment in rt6_mtu_change_route()]
         */

        if (iph) {
                daddr = &iph->daddr;
                saddr = &iph->saddr;
        } else if (sk) {
                daddr = &sk->sk_v6_daddr;
                saddr = &inet6_sk(sk)->saddr;
        } else {
                daddr = NULL;
                saddr = NULL;
        }

        if (confirm_neigh)
                dst_confirm_neigh(dst, daddr);

        if (mtu < IPV6_MIN_MTU)
                return;
        if (mtu >= dst_mtu(dst))
                return;

        if (!rt6_cache_allowed_for_pmtu(rt6)) {
                rt6_do_update_pmtu(rt6, mtu);
                /* update rt6_ex->stamp for cache */
                if (rt6->rt6i_flags & RTF_CACHE)
                        rt6_update_exception_stamp_rt(rt6);
        } else if (daddr) {
                struct fib6_result res = {};
                struct rt6_info *nrt6;

                rcu_read_lock();
                res.f6i = rcu_dereference(rt6->from);
                if (!res.f6i)
                        goto out_unlock;

                res.fib6_flags = res.f6i->fib6_flags;
                res.fib6_type = res.f6i->fib6_type;

                if (res.f6i->nh) {
                        struct fib6_nh_match_arg arg = {
                                .dev = dst->dev,
                                .gw = &rt6->rt6i_gateway,
                        };

                        nexthop_for_each_fib6_nh(res.f6i->nh,
                                                 fib6_nh_find_match, &arg);

                        /* fib6_info uses a nexthop that does not have fib6_nh
                         * using the dst->dev + gw. Should be impossible.
                         */
                        if (!arg.match)
                                goto out_unlock;

                        res.nh = arg.match;
                } else {
                        res.nh = res.f6i->fib6_nh;
                }

                nrt6 = ip6_rt_cache_alloc(&res, daddr, saddr);
                if (nrt6) {
                        rt6_do_update_pmtu(nrt6, mtu);
                        if (rt6_insert_exception(nrt6, &res))
                                dst_release_immediate(&nrt6->dst);
                }
out_unlock:
                rcu_read_unlock();
        }
}

static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
                               struct sk_buff *skb, u32 mtu,
                               bool confirm_neigh)
{
        __ip6_rt_update_pmtu(dst, sk, skb ? ipv6_hdr(skb) : NULL, mtu,
                             confirm_neigh);
}

void ip6_update_pmtu(struct sk_buff *skb, struct net *net, __be32 mtu,
                     int oif, u32 mark, kuid_t uid)
{
        const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data;
        struct dst_entry *dst;
        struct flowi6 fl6 = {
                .flowi6_oif = oif,
                .flowi6_mark = mark ? mark : IP6_REPLY_MARK(net, skb->mark),
                .daddr = iph->daddr,
                .saddr = iph->saddr,
                .flowlabel = ip6_flowinfo(iph),
                .flowi6_uid = uid,
        };

        dst = ip6_route_output(net, NULL, &fl6);
        if (!dst->error)
                __ip6_rt_update_pmtu(dst, NULL, iph, ntohl(mtu), true);
        dst_release(dst);
}
EXPORT_SYMBOL_GPL(ip6_update_pmtu);

void ip6_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, __be32 mtu)
{
        int oif = sk->sk_bound_dev_if;
        struct dst_entry *dst;

        if (!oif && skb->dev)
                oif = l3mdev_master_ifindex(skb->dev);

        ip6_update_pmtu(skb, sock_net(sk), mtu, oif, READ_ONCE(sk->sk_mark),
                        sk->sk_uid);

        dst = __sk_dst_get(sk);
        if (!dst || !dst->obsolete ||
            dst->ops->check(dst, inet6_sk(sk)->dst_cookie))
                return;

        bh_lock_sock(sk);
        if (!sock_owned_by_user(sk) && !ipv6_addr_v4mapped(&sk->sk_v6_daddr))
                ip6_datagram_dst_update(sk, false);
        bh_unlock_sock(sk);
}
EXPORT_SYMBOL_GPL(ip6_sk_update_pmtu);

void ip6_sk_dst_store_flow(struct sock *sk, struct dst_entry *dst,
                           const struct flowi6 *fl6)
{
#ifdef CONFIG_IPV6_SUBTREES
        struct ipv6_pinfo *np = inet6_sk(sk);
#endif

        ip6_dst_store(sk, dst,
                      ipv6_addr_equal(&fl6->daddr, &sk->sk_v6_daddr) ?
                      &sk->sk_v6_daddr : NULL,
#ifdef CONFIG_IPV6_SUBTREES
                      ipv6_addr_equal(&fl6->saddr, &np->saddr) ?
                      &np->saddr :
#endif
                      NULL);
}

static bool ip6_redirect_nh_match(const struct fib6_result *res,
                                  struct flowi6 *fl6,
                                  const struct in6_addr *gw,
                                  struct rt6_info **ret)
{
        const struct fib6_nh *nh = res->nh;

        if (nh->fib_nh_flags & RTNH_F_DEAD || !nh->fib_nh_gw_family ||
            fl6->flowi6_oif != nh->fib_nh_dev->ifindex)
                return false;

        /* rt_cache's gateway might be different from its 'parent'
         * in the case of an ip redirect.
         * So we keep searching in the exception table if the gateway
         * is different.
         */
        if (!ipv6_addr_equal(gw, &nh->fib_nh_gw6)) {
                struct rt6_info *rt_cache;

                rt_cache = rt6_find_cached_rt(res, &fl6->daddr, &fl6->saddr);
                if (rt_cache &&
                    ipv6_addr_equal(gw, &rt_cache->rt6i_gateway)) {
                        *ret = rt_cache;
                        return true;
                }
                return false;
        }
        return true;
}

struct fib6_nh_rd_arg {
        struct fib6_result        *res;
        struct flowi6                *fl6;
        const struct in6_addr        *gw;
        struct rt6_info                **ret;
};

static int fib6_nh_redirect_match(struct fib6_nh *nh, void *_arg)
{
        struct fib6_nh_rd_arg *arg = _arg;

        arg->res->nh = nh;
        return ip6_redirect_nh_match(arg->res, arg->fl6, arg->gw, arg->ret);
}

/* Handle redirects */
struct ip6rd_flowi {
        struct flowi6 fl6;
        struct in6_addr gateway;
};

INDIRECT_CALLABLE_SCOPE struct rt6_info *__ip6_route_redirect(struct net *net,
                                             struct fib6_table *table,
                                             struct flowi6 *fl6,
                                             const struct sk_buff *skb,
                                             int flags)
{
        struct ip6rd_flowi *rdfl = (struct ip6rd_flowi *)fl6;
        struct rt6_info *ret = NULL;
        struct fib6_result res = {};
        struct fib6_nh_rd_arg arg = {
                .res = &res,
                .fl6 = fl6,
                .gw  = &rdfl->gateway,
                .ret = &ret
        };
        struct fib6_info *rt;
        struct fib6_node *fn;

        /* Get the "current" route for this destination and
         * check if the redirect has come from appropriate router.
         *
         * RFC 4861 specifies that redirects should only be
         * accepted if they come from the nexthop to the target.
         * Due to the way the routes are chosen, this notion
         * is a bit fuzzy and one might need to check all possible
         * routes.
         */

        rcu_read_lock();
        fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr);
restart:
        for_each_fib6_node_rt_rcu(fn) {
                res.f6i = rt;
                if (fib6_check_expired(rt))
                        continue;
                if (rt->fib6_flags & RTF_REJECT)
                        break;
                if (unlikely(rt->nh)) {
                        if (nexthop_is_blackhole(rt->nh))
                                continue;
                        /* on match, res->nh is filled in and potentially ret */
                        if (nexthop_for_each_fib6_nh(rt->nh,
                                                     fib6_nh_redirect_match,
                                                     &arg))
                                goto out;
                } else {
                        res.nh = rt->fib6_nh;
                        if (ip6_redirect_nh_match(&res, fl6, &rdfl->gateway,
                                                  &ret))
                                goto out;
                }
        }

        if (!rt)
                rt = net->ipv6.fib6_null_entry;
        else if (rt->fib6_flags & RTF_REJECT) {
                ret = net->ipv6.ip6_null_entry;
                goto out;
        }

        if (rt == net->ipv6.fib6_null_entry) {
                fn = fib6_backtrack(fn, &fl6->saddr);
                if (fn)
                        goto restart;
        }

        res.f6i = rt;
        res.nh = rt->fib6_nh;
out:
        if (ret) {
                ip6_hold_safe(net, &ret);
        } else {
                res.fib6_flags = res.f6i->fib6_flags;
                res.fib6_type = res.f6i->fib6_type;
                ret = ip6_create_rt_rcu(&res);
        }

        rcu_read_unlock();

        trace_fib6_table_lookup(net, &res, table, fl6);
        return ret;
};

static struct dst_entry *ip6_route_redirect(struct net *net,
                                            const struct flowi6 *fl6,
                                            const struct sk_buff *skb,
                                            const struct in6_addr *gateway)
{
        int flags = RT6_LOOKUP_F_HAS_SADDR;
        struct ip6rd_flowi rdfl;

        rdfl.fl6 = *fl6;
        rdfl.gateway = *gateway;

        return fib6_rule_lookup(net, &rdfl.fl6, skb,
                                flags, __ip6_route_redirect);
}

void ip6_redirect(struct sk_buff *skb, struct net *net, int oif, u32 mark,
                  kuid_t uid)
{
        const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data;
        struct dst_entry *dst;
        struct flowi6 fl6 = {
                .flowi6_iif = LOOPBACK_IFINDEX,
                .flowi6_oif = oif,
                .flowi6_mark = mark,
                .daddr = iph->daddr,
                .saddr = iph->saddr,
                .flowlabel = ip6_flowinfo(iph),
                .flowi6_uid = uid,
        };

        dst = ip6_route_redirect(net, &fl6, skb, &ipv6_hdr(skb)->saddr);
        rt6_do_redirect(dst, NULL, skb);
        dst_release(dst);
}
EXPORT_SYMBOL_GPL(ip6_redirect);

void ip6_redirect_no_header(struct sk_buff *skb, struct net *net, int oif)
{
        const struct ipv6hdr *iph = ipv6_hdr(skb);
        const struct rd_msg *msg = (struct rd_msg *)icmp6_hdr(skb);
        struct dst_entry *dst;
        struct flowi6 fl6 = {
                .flowi6_iif = LOOPBACK_IFINDEX,
                .flowi6_oif = oif,
                .daddr = msg->dest,
                .saddr = iph->daddr,
                .flowi6_uid = sock_net_uid(net, NULL),
        };

        dst = ip6_route_redirect(net, &fl6, skb, &iph->saddr);
        rt6_do_redirect(dst, NULL, skb);
        dst_release(dst);
}

void ip6_sk_redirect(struct sk_buff *skb, struct sock *sk)
{
        ip6_redirect(skb, sock_net(sk), sk->sk_bound_dev_if,
                     READ_ONCE(sk->sk_mark), sk->sk_uid);
}
EXPORT_SYMBOL_GPL(ip6_sk_redirect);

static unsigned int ip6_default_advmss(const struct dst_entry *dst)
{
        struct net_device *dev = dst->dev;
        unsigned int mtu = dst_mtu(dst);
        struct net *net;

        mtu -= sizeof(struct ipv6hdr) + sizeof(struct tcphdr);

        rcu_read_lock();

        net = dev_net_rcu(dev);
        if (mtu < net->ipv6.sysctl.ip6_rt_min_advmss)
                mtu = net->ipv6.sysctl.ip6_rt_min_advmss;

        rcu_read_unlock();

        /*
         * Maximal non-jumbo IPv6 payload is IPV6_MAXPLEN and
         * corresponding MSS is IPV6_MAXPLEN - tcp_header_size.
         * IPV6_MAXPLEN is also valid and means: "any MSS,
         * rely only on pmtu discovery"
         */
        if (mtu > IPV6_MAXPLEN - sizeof(struct tcphdr))
                mtu = IPV6_MAXPLEN;
        return mtu;
}

INDIRECT_CALLABLE_SCOPE unsigned int ip6_mtu(const struct dst_entry *dst)
{
        return ip6_dst_mtu_maybe_forward(dst, false);
}
EXPORT_INDIRECT_CALLABLE(ip6_mtu);

/* MTU selection:
 * 1. mtu on route is locked - use it
 * 2. mtu from nexthop exception
 * 3. mtu from egress device
 *
 * based on ip6_dst_mtu_forward and exception logic of
 * rt6_find_cached_rt; called with rcu_read_lock
 */
u32 ip6_mtu_from_fib6(const struct fib6_result *res,
                      const struct in6_addr *daddr,
                      const struct in6_addr *saddr)
{
        const struct fib6_nh *nh = res->nh;
        struct fib6_info *f6i = res->f6i;
        struct inet6_dev *idev;
        struct rt6_info *rt;
        u32 mtu = 0;

        if (unlikely(fib6_metric_locked(f6i, RTAX_MTU))) {
                mtu = f6i->fib6_pmtu;
                if (mtu)
                        goto out;
        }

        rt = rt6_find_cached_rt(res, daddr, saddr);
        if (unlikely(rt)) {
                mtu = dst_metric_raw(&rt->dst, RTAX_MTU);
        } else {
                struct net_device *dev = nh->fib_nh_dev;

                mtu = IPV6_MIN_MTU;
                idev = __in6_dev_get(dev);
                if (idev)
                        mtu = max_t(u32, mtu, READ_ONCE(idev->cnf.mtu6));
        }

        mtu = min_t(unsigned int, mtu, IP6_MAX_MTU);
out:
        return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu);
}

struct dst_entry *icmp6_dst_alloc(struct net_device *dev,
                                  struct flowi6 *fl6)
{
        struct dst_entry *dst;
        struct rt6_info *rt;
        struct inet6_dev *idev = in6_dev_get(dev);
        struct net *net = dev_net(dev);

        if (unlikely(!idev))
                return ERR_PTR(-ENODEV);

        rt = ip6_dst_alloc(net, dev, 0);
        if (unlikely(!rt)) {
                in6_dev_put(idev);
                dst = ERR_PTR(-ENOMEM);
                goto out;
        }

        rt->dst.input = ip6_input;
        rt->dst.output  = ip6_output;
        rt->rt6i_gateway  = fl6->daddr;
        rt->rt6i_dst.addr = fl6->daddr;
        rt->rt6i_dst.plen = 128;
        rt->rt6i_idev     = idev;
        dst_metric_set(&rt->dst, RTAX_HOPLIMIT, 0);

        /* Add this dst into uncached_list so that rt6_disable_ip() can
         * do proper release of the net_device
         */
        rt6_uncached_list_add(rt);

        dst = xfrm_lookup(net, &rt->dst, flowi6_to_flowi(fl6), NULL, 0);

out:
        return dst;
}

static void ip6_dst_gc(struct dst_ops *ops)
{
        struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops);
        int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval;
        int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity;
        int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout;
        unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;
        unsigned int val;
        int entries;

        if (time_after(rt_last_gc + rt_min_interval, jiffies))
                goto out;

        fib6_run_gc(atomic_inc_return(&net->ipv6.ip6_rt_gc_expire), net, true);
        entries = dst_entries_get_slow(ops);
        if (entries < ops->gc_thresh)
                atomic_set(&net->ipv6.ip6_rt_gc_expire, rt_gc_timeout >> 1);
out:
        val = atomic_read(&net->ipv6.ip6_rt_gc_expire);
        atomic_set(&net->ipv6.ip6_rt_gc_expire, val - (val >> rt_elasticity));
}

static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg,
                               const struct in6_addr *gw_addr, u32 tbid,
                               int flags, struct fib6_result *res)
{
        struct flowi6 fl6 = {
                .flowi6_oif = cfg->fc_ifindex,
                .daddr = *gw_addr,
                .saddr = cfg->fc_prefsrc,
        };
        struct fib6_table *table;
        int err;

        table = fib6_get_table(net, tbid);
        if (!table)
                return -EINVAL;

        if (!ipv6_addr_any(&cfg->fc_prefsrc))
                flags |= RT6_LOOKUP_F_HAS_SADDR;

        flags |= RT6_LOOKUP_F_IGNORE_LINKSTATE;

        err = fib6_table_lookup(net, table, cfg->fc_ifindex, &fl6, res, flags);
        if (!err && res->f6i != net->ipv6.fib6_null_entry)
                fib6_select_path(net, res, &fl6, cfg->fc_ifindex,
                                 cfg->fc_ifindex != 0, NULL, flags);

        return err;
}

static int ip6_route_check_nh_onlink(struct net *net,
                                     struct fib6_config *cfg,
                                     const struct net_device *dev,
                                     struct netlink_ext_ack *extack)
{
        u32 tbid = l3mdev_fib_table_rcu(dev) ? : RT_TABLE_MAIN;
        const struct in6_addr *gw_addr = &cfg->fc_gateway;
        struct fib6_result res = {};
        int err;

        err = ip6_nh_lookup_table(net, cfg, gw_addr, tbid, 0, &res);
        if (!err && !(res.fib6_flags & RTF_REJECT) &&
            /* ignore match if it is the default route */
            !ipv6_addr_any(&res.f6i->fib6_dst.addr) &&
            (res.fib6_type != RTN_UNICAST || dev != res.nh->fib_nh_dev)) {
                NL_SET_ERR_MSG(extack,
                               "Nexthop has invalid gateway or device mismatch");
                err = -EINVAL;
        }

        return err;
}

static int ip6_route_check_nh(struct net *net,
                              struct fib6_config *cfg,
                              struct net_device **_dev,
                              netdevice_tracker *dev_tracker,
                              struct inet6_dev **idev)
{
        const struct in6_addr *gw_addr = &cfg->fc_gateway;
        struct net_device *dev = _dev ? *_dev : NULL;
        int flags = RT6_LOOKUP_F_IFACE;
        struct fib6_result res = {};
        int err = -EHOSTUNREACH;

        if (cfg->fc_table) {
                err = ip6_nh_lookup_table(net, cfg, gw_addr,
                                          cfg->fc_table, flags, &res);
                /* gw_addr can not require a gateway or resolve to a reject
                 * route. If a device is given, it must match the result.
                 */
                if (err || res.fib6_flags & RTF_REJECT ||
                    res.nh->fib_nh_gw_family ||
                    (dev && dev != res.nh->fib_nh_dev))
                        err = -EHOSTUNREACH;
        }

        if (err < 0) {
                struct flowi6 fl6 = {
                        .flowi6_oif = cfg->fc_ifindex,
                        .daddr = *gw_addr,
                };

                err = fib6_lookup(net, cfg->fc_ifindex, &fl6, &res, flags);
                if (err || res.fib6_flags & RTF_REJECT ||
                    res.nh->fib_nh_gw_family)
                        err = -EHOSTUNREACH;

                if (err)
                        return err;

                fib6_select_path(net, &res, &fl6, cfg->fc_ifindex,
                                 cfg->fc_ifindex != 0, NULL, flags);
        }

        err = 0;
        if (dev) {
                if (dev != res.nh->fib_nh_dev)
                        err = -EHOSTUNREACH;
        } else {
                *_dev = dev = res.nh->fib_nh_dev;
                netdev_hold(dev, dev_tracker, GFP_ATOMIC);
                *idev = in6_dev_get(dev);
        }

        return err;
}

static int ip6_validate_gw(struct net *net, struct fib6_config *cfg,
                           struct net_device **_dev,
                           netdevice_tracker *dev_tracker,
                           struct inet6_dev **idev,
                           struct netlink_ext_ack *extack)
{
        const struct in6_addr *gw_addr = &cfg->fc_gateway;
        int gwa_type = ipv6_addr_type(gw_addr);
        bool skip_dev = gwa_type & IPV6_ADDR_LINKLOCAL ? false : true;
        const struct net_device *dev = *_dev;
        bool need_addr_check = !dev;
        int err = -EINVAL;

        /* if gw_addr is local we will fail to detect this in case
         * address is still TENTATIVE (DAD in progress). rt6_lookup()
         * will return already-added prefix route via interface that
         * prefix route was assigned to, which might be non-loopback.
         */
        if (dev &&
            ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) {
                NL_SET_ERR_MSG(extack, "Gateway can not be a local address");
                goto out;
        }

        if (gwa_type != (IPV6_ADDR_LINKLOCAL | IPV6_ADDR_UNICAST)) {
                /* IPv6 strictly inhibits using not link-local
                 * addresses as nexthop address.
                 * Otherwise, router will not able to send redirects.
                 * It is very good, but in some (rare!) circumstances
                 * (SIT, PtP, NBMA NOARP links) it is handy to allow
                 * some exceptions. --ANK
                 * We allow IPv4-mapped nexthops to support RFC4798-type
                 * addressing
                 */
                if (!(gwa_type & (IPV6_ADDR_UNICAST | IPV6_ADDR_MAPPED))) {
                        NL_SET_ERR_MSG(extack, "Invalid gateway address");
                        goto out;
                }

                rcu_read_lock();

                if (cfg->fc_flags & RTNH_F_ONLINK)
                        err = ip6_route_check_nh_onlink(net, cfg, dev, extack);
                else
                        err = ip6_route_check_nh(net, cfg, _dev, dev_tracker,
                                                 idev);

                rcu_read_unlock();

                if (err)
                        goto out;
        }

        /* reload in case device was changed */
        dev = *_dev;

        err = -EINVAL;
        if (!dev) {
                NL_SET_ERR_MSG(extack, "Egress device not specified");
                goto out;
        } else if (dev->flags & IFF_LOOPBACK) {
                NL_SET_ERR_MSG(extack,
                               "Egress device can not be loopback device for this route");
                goto out;
        }

        /* if we did not check gw_addr above, do so now that the
         * egress device has been resolved.
         */
        if (need_addr_check &&
            ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) {
                NL_SET_ERR_MSG(extack, "Gateway can not be a local address");
                goto out;
        }

        err = 0;
out:
        return err;
}

static bool fib6_is_reject(u32 flags, struct net_device *dev, int addr_type)
{
        if ((flags & RTF_REJECT) ||
            (dev && (dev->flags & IFF_LOOPBACK) &&
             !(addr_type & IPV6_ADDR_LOOPBACK) &&
             !(flags & (RTF_ANYCAST | RTF_LOCAL))))
                return true;

        return false;
}

int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh,
                 struct fib6_config *cfg, gfp_t gfp_flags,
                 struct netlink_ext_ack *extack)
{
        netdevice_tracker *dev_tracker = &fib6_nh->fib_nh_dev_tracker;
        struct net_device *dev = NULL;
        struct inet6_dev *idev = NULL;
        int addr_type;
        int err;

        fib6_nh->fib_nh_family = AF_INET6;
#ifdef CONFIG_IPV6_ROUTER_PREF
        fib6_nh->last_probe = jiffies;
#endif
        if (cfg->fc_is_fdb) {
                fib6_nh->fib_nh_gw6 = cfg->fc_gateway;
                fib6_nh->fib_nh_gw_family = AF_INET6;
                return 0;
        }

        err = -ENODEV;
        if (cfg->fc_ifindex) {
                dev = netdev_get_by_index(net, cfg->fc_ifindex,
                                          dev_tracker, gfp_flags);
                if (!dev)
                        goto out;
                idev = in6_dev_get(dev);
                if (!idev)
                        goto out;
        }

        if (cfg->fc_flags & RTNH_F_ONLINK) {
                if (!dev) {
                        NL_SET_ERR_MSG(extack,
                                       "Nexthop device required for onlink");
                        goto out;
                }

                if (!(dev->flags & IFF_UP)) {
                        NL_SET_ERR_MSG(extack, "Nexthop device is not up");
                        err = -ENETDOWN;
                        goto out;
                }

                fib6_nh->fib_nh_flags |= RTNH_F_ONLINK;
        }

        fib6_nh->fib_nh_weight = 1;

        /* We cannot add true routes via loopback here,
         * they would result in kernel looping; promote them to reject routes
         */
        addr_type = ipv6_addr_type(&cfg->fc_dst);
        if (fib6_is_reject(cfg->fc_flags, dev, addr_type)) {
                /* hold loopback dev/idev if we haven't done so. */
                if (dev != net->loopback_dev) {
                        if (dev) {
                                netdev_put(dev, dev_tracker);
                                in6_dev_put(idev);
                        }
                        dev = net->loopback_dev;
                        netdev_hold(dev, dev_tracker, gfp_flags);
                        idev = in6_dev_get(dev);
                        if (!idev) {
                                err = -ENODEV;
                                goto out;
                        }
                }
                goto pcpu_alloc;
        }

        if (cfg->fc_flags & RTF_GATEWAY) {
                err = ip6_validate_gw(net, cfg, &dev, dev_tracker,
                                      &idev, extack);
                if (err)
                        goto out;

                fib6_nh->fib_nh_gw6 = cfg->fc_gateway;
                fib6_nh->fib_nh_gw_family = AF_INET6;
        }

        err = -ENODEV;
        if (!dev)
                goto out;

        if (!idev || idev->cnf.disable_ipv6) {
                NL_SET_ERR_MSG(extack, "IPv6 is disabled on nexthop device");
                err = -EACCES;
                goto out;
        }

        if (!(dev->flags & IFF_UP) && !cfg->fc_ignore_dev_down) {
                NL_SET_ERR_MSG(extack, "Nexthop device is not up");
                err = -ENETDOWN;
                goto out;
        }

        if (!(cfg->fc_flags & (RTF_LOCAL | RTF_ANYCAST)) &&
            !netif_carrier_ok(dev))
                fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN;

        err = fib_nh_common_init(net, &fib6_nh->nh_common, cfg->fc_encap,
                                 cfg->fc_encap_type, cfg, gfp_flags, extack);
        if (err)
                goto out;

pcpu_alloc:
        fib6_nh->rt6i_pcpu = alloc_percpu_gfp(struct rt6_info *, gfp_flags);
        if (!fib6_nh->rt6i_pcpu) {
                err = -ENOMEM;
                goto out;
        }

        fib6_nh->fib_nh_dev = dev;
        fib6_nh->fib_nh_oif = dev->ifindex;
        err = 0;
out:
        if (idev)
                in6_dev_put(idev);

        if (err) {
                fib_nh_common_release(&fib6_nh->nh_common);
                fib6_nh->nh_common.nhc_pcpu_rth_output = NULL;
                fib6_nh->fib_nh_lws = NULL;
                netdev_put(dev, dev_tracker);
        }

        return err;
}

void fib6_nh_release(struct fib6_nh *fib6_nh)
{
        struct rt6_exception_bucket *bucket;

        rcu_read_lock();

        fib6_nh_flush_exceptions(fib6_nh, NULL);
        bucket = fib6_nh_get_excptn_bucket(fib6_nh, NULL);
        if (bucket) {
                rcu_assign_pointer(fib6_nh->rt6i_exception_bucket, NULL);
                kfree(bucket);
        }

        rcu_read_unlock();

        fib6_nh_release_dsts(fib6_nh);
        free_percpu(fib6_nh->rt6i_pcpu);

        fib_nh_common_release(&fib6_nh->nh_common);
}

void fib6_nh_release_dsts(struct fib6_nh *fib6_nh)
{
        int cpu;

        if (!fib6_nh->rt6i_pcpu)
                return;

        for_each_possible_cpu(cpu) {
                struct rt6_info *pcpu_rt, **ppcpu_rt;

                ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu);
                pcpu_rt = xchg(ppcpu_rt, NULL);
                if (pcpu_rt) {
                        dst_dev_put(&pcpu_rt->dst);
                        dst_release(&pcpu_rt->dst);
                }
        }
}

static struct fib6_info *ip6_route_info_create(struct fib6_config *cfg,
                                              gfp_t gfp_flags,
                                              struct netlink_ext_ack *extack)
{
        struct net *net = cfg->fc_nlinfo.nl_net;
        struct fib6_info *rt = NULL;
        struct nexthop *nh = NULL;
        struct fib6_table *table;
        struct fib6_nh *fib6_nh;
        int err = -EINVAL;
        int addr_type;

        /* RTF_PCPU is an internal flag; can not be set by userspace */
        if (cfg->fc_flags & RTF_PCPU) {
                NL_SET_ERR_MSG(extack, "Userspace can not set RTF_PCPU");
                goto out;
        }

        /* RTF_CACHE is an internal flag; can not be set by userspace */
        if (cfg->fc_flags & RTF_CACHE) {
                NL_SET_ERR_MSG(extack, "Userspace can not set RTF_CACHE");
                goto out;
        }

        if (cfg->fc_type > RTN_MAX) {
                NL_SET_ERR_MSG(extack, "Invalid route type");
                goto out;
        }

        if (cfg->fc_dst_len > 128) {
                NL_SET_ERR_MSG(extack, "Invalid prefix length");
                goto out;
        }
        if (cfg->fc_src_len > 128) {
                NL_SET_ERR_MSG(extack, "Invalid source address length");
                goto out;
        }
#ifndef CONFIG_IPV6_SUBTREES
        if (cfg->fc_src_len) {
                NL_SET_ERR_MSG(extack,
                               "Specifying source address requires IPV6_SUBTREES to be enabled");
                goto out;
        }
#endif
        if (cfg->fc_nh_id) {
                nh = nexthop_find_by_id(net, cfg->fc_nh_id);
                if (!nh) {
                        NL_SET_ERR_MSG(extack, "Nexthop id does not exist");
                        goto out;
                }
                err = fib6_check_nexthop(nh, cfg, extack);
                if (err)
                        goto out;
        }

        err = -ENOBUFS;
        if (cfg->fc_nlinfo.nlh &&
            !(cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_CREATE)) {
                table = fib6_get_table(net, cfg->fc_table);
                if (!table) {
                        pr_warn("NLM_F_CREATE should be specified when creating new route\n");
                        table = fib6_new_table(net, cfg->fc_table);
                }
        } else {
                table = fib6_new_table(net, cfg->fc_table);
        }

        if (!table)
                goto out;

        err = -ENOMEM;
        rt = fib6_info_alloc(gfp_flags, !nh);
        if (!rt)
                goto out;

        rt->fib6_metrics = ip_fib_metrics_init(cfg->fc_mx, cfg->fc_mx_len,
                                               extack);
        if (IS_ERR(rt->fib6_metrics)) {
                err = PTR_ERR(rt->fib6_metrics);
                /* Do not leave garbage there. */
                rt->fib6_metrics = (struct dst_metrics *)&dst_default_metrics;
                goto out_free;
        }

        if (cfg->fc_flags & RTF_ADDRCONF)
                rt->dst_nocount = true;

        if (cfg->fc_flags & RTF_EXPIRES)
                fib6_set_expires(rt, jiffies +
                                clock_t_to_jiffies(cfg->fc_expires));

        if (cfg->fc_protocol == RTPROT_UNSPEC)
                cfg->fc_protocol = RTPROT_BOOT;
        rt->fib6_protocol = cfg->fc_protocol;

        rt->fib6_table = table;
        rt->fib6_metric = cfg->fc_metric;
        rt->fib6_type = cfg->fc_type ? : RTN_UNICAST;
        rt->fib6_flags = cfg->fc_flags & ~RTF_GATEWAY;

        ipv6_addr_prefix(&rt->fib6_dst.addr, &cfg->fc_dst, cfg->fc_dst_len);
        rt->fib6_dst.plen = cfg->fc_dst_len;

#ifdef CONFIG_IPV6_SUBTREES
        ipv6_addr_prefix(&rt->fib6_src.addr, &cfg->fc_src, cfg->fc_src_len);
        rt->fib6_src.plen = cfg->fc_src_len;
#endif
        if (nh) {
                if (rt->fib6_src.plen) {
                        NL_SET_ERR_MSG(extack, "Nexthops can not be used with source routing");
                        err = -EINVAL;
                        goto out_free;
                }
                if (!nexthop_get(nh)) {
                        NL_SET_ERR_MSG(extack, "Nexthop has been deleted");
                        err = -ENOENT;
                        goto out_free;
                }
                rt->nh = nh;
                fib6_nh = nexthop_fib6_nh(rt->nh);
        } else {
                err = fib6_nh_init(net, rt->fib6_nh, cfg, gfp_flags, extack);
                if (err)
                        goto out;

                fib6_nh = rt->fib6_nh;

                /* We cannot add true routes via loopback here, they would
                 * result in kernel looping; promote them to reject routes
                 */
                addr_type = ipv6_addr_type(&cfg->fc_dst);
                if (fib6_is_reject(cfg->fc_flags, rt->fib6_nh->fib_nh_dev,
                                   addr_type))
                        rt->fib6_flags = RTF_REJECT | RTF_NONEXTHOP;
        }

        if (!ipv6_addr_any(&cfg->fc_prefsrc)) {
                struct net_device *dev = fib6_nh->fib_nh_dev;

                if (!ipv6_chk_addr(net, &cfg->fc_prefsrc, dev, 0)) {
                        NL_SET_ERR_MSG(extack, "Invalid source address");
                        err = -EINVAL;
                        goto out;
                }
                rt->fib6_prefsrc.addr = cfg->fc_prefsrc;
                rt->fib6_prefsrc.plen = 128;
        } else
                rt->fib6_prefsrc.plen = 0;

        return rt;
out:
        fib6_info_release(rt);
        return ERR_PTR(err);
out_free:
        ip_fib_metrics_put(rt->fib6_metrics);
        kfree(rt);
        return ERR_PTR(err);
}

int ip6_route_add(struct fib6_config *cfg, gfp_t gfp_flags,
                  struct netlink_ext_ack *extack)
{
        struct fib6_info *rt;
        int err;

        rt = ip6_route_info_create(cfg, gfp_flags, extack);
        if (IS_ERR(rt))
                return PTR_ERR(rt);

        err = __ip6_ins_rt(rt, &cfg->fc_nlinfo, extack);
        fib6_info_release(rt);

        return err;
}

static int __ip6_del_rt(struct fib6_info *rt, struct nl_info *info)
{
        struct net *net = info->nl_net;
        struct fib6_table *table;
        int err;

        if (rt == net->ipv6.fib6_null_entry) {
                err = -ENOENT;
                goto out;
        }

        table = rt->fib6_table;
        spin_lock_bh(&table->tb6_lock);
        err = fib6_del(rt, info);
        spin_unlock_bh(&table->tb6_lock);

out:
        fib6_info_release(rt);
        return err;
}

int ip6_del_rt(struct net *net, struct fib6_info *rt, bool skip_notify)
{
        struct nl_info info = {
                .nl_net = net,
                .skip_notify = skip_notify
        };

        return __ip6_del_rt(rt, &info);
}

static int __ip6_del_rt_siblings(struct fib6_info *rt, struct fib6_config *cfg)
{
        struct nl_info *info = &cfg->fc_nlinfo;
        struct net *net = info->nl_net;
        struct sk_buff *skb = NULL;
        struct fib6_table *table;
        int err = -ENOENT;

        if (rt == net->ipv6.fib6_null_entry)
                goto out_put;
        table = rt->fib6_table;
        spin_lock_bh(&table->tb6_lock);

        if (rt->fib6_nsiblings && cfg->fc_delete_all_nh) {
                struct fib6_info *sibling, *next_sibling;
                struct fib6_node *fn;

                /* prefer to send a single notification with all hops */
                skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any());
                if (skb) {
                        u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;

                        if (rt6_fill_node(net, skb, rt, NULL,
                                          NULL, NULL, 0, RTM_DELROUTE,
                                          info->portid, seq, 0) < 0) {
                                kfree_skb(skb);
                                skb = NULL;
                        } else
                                info->skip_notify = 1;
                }

                /* 'rt' points to the first sibling route. If it is not the
                 * leaf, then we do not need to send a notification. Otherwise,
                 * we need to check if the last sibling has a next route or not
                 * and emit a replace or delete notification, respectively.
                 */
                info->skip_notify_kernel = 1;
                fn = rcu_dereference_protected(rt->fib6_node,
                                            lockdep_is_held(&table->tb6_lock));
                if (rcu_access_pointer(fn->leaf) == rt) {
                        struct fib6_info *last_sibling, *replace_rt;

                        last_sibling = list_last_entry(&rt->fib6_siblings,
                                                       struct fib6_info,
                                                       fib6_siblings);
                        replace_rt = rcu_dereference_protected(
                                            last_sibling->fib6_next,
                                            lockdep_is_held(&table->tb6_lock));
                        if (replace_rt)
                                call_fib6_entry_notifiers_replace(net,
                                                                  replace_rt);
                        else
                                call_fib6_multipath_entry_notifiers(net,
                                                       FIB_EVENT_ENTRY_DEL,
                                                       rt, rt->fib6_nsiblings,
                                                       NULL);
                }
                list_for_each_entry_safe(sibling, next_sibling,
                                         &rt->fib6_siblings,
                                         fib6_siblings) {
                        err = fib6_del(sibling, info);
                        if (err)
                                goto out_unlock;
                }
        }

        err = fib6_del(rt, info);
out_unlock:
        spin_unlock_bh(&table->tb6_lock);
out_put:
        fib6_info_release(rt);

        if (skb) {
                rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE,
                            info->nlh, gfp_any());
        }
        return err;
}

static int __ip6_del_cached_rt(struct rt6_info *rt, struct fib6_config *cfg)
{
        int rc = -ESRCH;

        if (cfg->fc_ifindex && rt->dst.dev->ifindex != cfg->fc_ifindex)
                goto out;

        if (cfg->fc_flags & RTF_GATEWAY &&
            !ipv6_addr_equal(&cfg->fc_gateway, &rt->rt6i_gateway))
                goto out;

        rc = rt6_remove_exception_rt(rt);
out:
        return rc;
}

static int ip6_del_cached_rt(struct fib6_config *cfg, struct fib6_info *rt,
                             struct fib6_nh *nh)
{
        struct fib6_result res = {
                .f6i = rt,
                .nh = nh,
        };
        struct rt6_info *rt_cache;

        rt_cache = rt6_find_cached_rt(&res, &cfg->fc_dst, &cfg->fc_src);
        if (rt_cache)
                return __ip6_del_cached_rt(rt_cache, cfg);

        return 0;
}

struct fib6_nh_del_cached_rt_arg {
        struct fib6_config *cfg;
        struct fib6_info *f6i;
};

static int fib6_nh_del_cached_rt(struct fib6_nh *nh, void *_arg)
{
        struct fib6_nh_del_cached_rt_arg *arg = _arg;
        int rc;

        rc = ip6_del_cached_rt(arg->cfg, arg->f6i, nh);
        return rc != -ESRCH ? rc : 0;
}

static int ip6_del_cached_rt_nh(struct fib6_config *cfg, struct fib6_info *f6i)
{
        struct fib6_nh_del_cached_rt_arg arg = {
                .cfg = cfg,
                .f6i = f6i
        };

        return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_del_cached_rt, &arg);
}

static int ip6_route_del(struct fib6_config *cfg,
                         struct netlink_ext_ack *extack)
{
        struct fib6_table *table;
        struct fib6_info *rt;
        struct fib6_node *fn;
        int err = -ESRCH;

        table = fib6_get_table(cfg->fc_nlinfo.nl_net, cfg->fc_table);
        if (!table) {
                NL_SET_ERR_MSG(extack, "FIB table does not exist");
                return err;
        }

        rcu_read_lock();

        fn = fib6_locate(&table->tb6_root,
                         &cfg->fc_dst, cfg->fc_dst_len,
                         &cfg->fc_src, cfg->fc_src_len,
                         !(cfg->fc_flags & RTF_CACHE));

        if (fn) {
                for_each_fib6_node_rt_rcu(fn) {
                        struct fib6_nh *nh;

                        if (rt->nh && cfg->fc_nh_id &&
                            rt->nh->id != cfg->fc_nh_id)
                                continue;

                        if (cfg->fc_flags & RTF_CACHE) {
                                int rc = 0;

                                if (rt->nh) {
                                        rc = ip6_del_cached_rt_nh(cfg, rt);
                                } else if (cfg->fc_nh_id) {
                                        continue;
                                } else {
                                        nh = rt->fib6_nh;
                                        rc = ip6_del_cached_rt(cfg, rt, nh);
                                }
                                if (rc != -ESRCH) {
                                        rcu_read_unlock();
                                        return rc;
                                }
                                continue;
                        }

                        if (cfg->fc_metric && cfg->fc_metric != rt->fib6_metric)
                                continue;
                        if (cfg->fc_protocol &&
                            cfg->fc_protocol != rt->fib6_protocol)
                                continue;

                        if (rt->nh) {
                                if (!fib6_info_hold_safe(rt))
                                        continue;
                                rcu_read_unlock();

                                return __ip6_del_rt(rt, &cfg->fc_nlinfo);
                        }
                        if (cfg->fc_nh_id)
                                continue;

                        nh = rt->fib6_nh;
                        if (cfg->fc_ifindex &&
                            (!nh->fib_nh_dev ||
                             nh->fib_nh_dev->ifindex != cfg->fc_ifindex))
                                continue;
                        if (cfg->fc_flags & RTF_GATEWAY &&
                            !ipv6_addr_equal(&cfg->fc_gateway, &nh->fib_nh_gw6))
                                continue;
                        if (!fib6_info_hold_safe(rt))
                                continue;
                        rcu_read_unlock();

                        /* if gateway was specified only delete the one hop */
                        if (cfg->fc_flags & RTF_GATEWAY)
                                return __ip6_del_rt(rt, &cfg->fc_nlinfo);

                        return __ip6_del_rt_siblings(rt, cfg);
                }
        }
        rcu_read_unlock();

        return err;
}

static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
{
        struct netevent_redirect netevent;
        struct rt6_info *rt, *nrt = NULL;
        struct fib6_result res = {};
        struct ndisc_options ndopts;
        struct inet6_dev *in6_dev;
        struct neighbour *neigh;
        struct rd_msg *msg;
        int optlen, on_link;
        u8 *lladdr;

        optlen = skb_tail_pointer(skb) - skb_transport_header(skb);
        optlen -= sizeof(*msg);

        if (optlen < 0) {
                net_dbg_ratelimited("rt6_do_redirect: packet too short\n");
                return;
        }

        msg = (struct rd_msg *)icmp6_hdr(skb);

        if (ipv6_addr_is_multicast(&msg->dest)) {
                net_dbg_ratelimited("rt6_do_redirect: destination address is multicast\n");
                return;
        }

        on_link = 0;
        if (ipv6_addr_equal(&msg->dest, &msg->target)) {
                on_link = 1;
        } else if (ipv6_addr_type(&msg->target) !=
                   (IPV6_ADDR_UNICAST|IPV6_ADDR_LINKLOCAL)) {
                net_dbg_ratelimited("rt6_do_redirect: target address is not link-local unicast\n");
                return;
        }

        in6_dev = __in6_dev_get(skb->dev);
        if (!in6_dev)
                return;
        if (READ_ONCE(in6_dev->cnf.forwarding) ||
            !READ_ONCE(in6_dev->cnf.accept_redirects))
                return;

        /* RFC2461 8.1:
         *        The IP source address of the Redirect MUST be the same as the current
         *        first-hop router for the specified ICMP Destination Address.
         */

        if (!ndisc_parse_options(skb->dev, msg->opt, optlen, &ndopts)) {
                net_dbg_ratelimited("rt6_redirect: invalid ND options\n");
                return;
        }

        lladdr = NULL;
        if (ndopts.nd_opts_tgt_lladdr) {
                lladdr = ndisc_opt_addr_data(ndopts.nd_opts_tgt_lladdr,
                                             skb->dev);
                if (!lladdr) {
                        net_dbg_ratelimited("rt6_redirect: invalid link-layer address length\n");
                        return;
                }
        }

        rt = dst_rt6_info(dst);
        if (rt->rt6i_flags & RTF_REJECT) {
                net_dbg_ratelimited("rt6_redirect: source isn't a valid nexthop for redirect target\n");
                return;
        }

        /* Redirect received -> path was valid.
         * Look, redirects are sent only in response to data packets,
         * so that this nexthop apparently is reachable. --ANK
         */
        dst_confirm_neigh(&rt->dst, &ipv6_hdr(skb)->saddr);

        neigh = __neigh_lookup(&nd_tbl, &msg->target, skb->dev, 1);
        if (!neigh)
                return;

        /*
         *        We have finally decided to accept it.
         */

        ndisc_update(skb->dev, neigh, lladdr, NUD_STALE,
                     NEIGH_UPDATE_F_WEAK_OVERRIDE|
                     NEIGH_UPDATE_F_OVERRIDE|
                     (on_link ? 0 : (NEIGH_UPDATE_F_OVERRIDE_ISROUTER|
                                     NEIGH_UPDATE_F_ISROUTER)),
                     NDISC_REDIRECT, &ndopts);

        rcu_read_lock();
        res.f6i = rcu_dereference(rt->from);
        if (!res.f6i)
                goto out;

        if (res.f6i->nh) {
                struct fib6_nh_match_arg arg = {
                        .dev = dst->dev,
                        .gw = &rt->rt6i_gateway,
                };

                nexthop_for_each_fib6_nh(res.f6i->nh,
                                         fib6_nh_find_match, &arg);

                /* fib6_info uses a nexthop that does not have fib6_nh
                 * using the dst->dev. Should be impossible
                 */
                if (!arg.match)
                        goto out;
                res.nh = arg.match;
        } else {
                res.nh = res.f6i->fib6_nh;
        }

        res.fib6_flags = res.f6i->fib6_flags;
        res.fib6_type = res.f6i->fib6_type;
        nrt = ip6_rt_cache_alloc(&res, &msg->dest, NULL);
        if (!nrt)
                goto out;

        nrt->rt6i_flags = RTF_GATEWAY|RTF_UP|RTF_DYNAMIC|RTF_CACHE;
        if (on_link)
                nrt->rt6i_flags &= ~RTF_GATEWAY;

        nrt->rt6i_gateway = *(struct in6_addr *)neigh->primary_key;

        /* rt6_insert_exception() will take care of duplicated exceptions */
        if (rt6_insert_exception(nrt, &res)) {
                dst_release_immediate(&nrt->dst);
                goto out;
        }

        netevent.old = &rt->dst;
        netevent.new = &nrt->dst;
        netevent.daddr = &msg->dest;
        netevent.neigh = neigh;
        call_netevent_notifiers(NETEVENT_REDIRECT, &netevent);

out:
        rcu_read_unlock();
        neigh_release(neigh);
}

#ifdef CONFIG_IPV6_ROUTE_INFO
static struct fib6_info *rt6_get_route_info(struct net *net,
                                           const struct in6_addr *prefix, int prefixlen,
                                           const struct in6_addr *gwaddr,
                                           struct net_device *dev)
{
        u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO;
        int ifindex = dev->ifindex;
        struct fib6_node *fn;
        struct fib6_info *rt = NULL;
        struct fib6_table *table;

        table = fib6_get_table(net, tb_id);
        if (!table)
                return NULL;

        rcu_read_lock();
        fn = fib6_locate(&table->tb6_root, prefix, prefixlen, NULL, 0, true);
        if (!fn)
                goto out;

        for_each_fib6_node_rt_rcu(fn) {
                /* these routes do not use nexthops */
                if (rt->nh)
                        continue;
                if (rt->fib6_nh->fib_nh_dev->ifindex != ifindex)
                        continue;
                if (!(rt->fib6_flags & RTF_ROUTEINFO) ||
                    !rt->fib6_nh->fib_nh_gw_family)
                        continue;
                if (!ipv6_addr_equal(&rt->fib6_nh->fib_nh_gw6, gwaddr))
                        continue;
                if (!fib6_info_hold_safe(rt))
                        continue;
                break;
        }
out:
        rcu_read_unlock();
        return rt;
}

static struct fib6_info *rt6_add_route_info(struct net *net,
                                           const struct in6_addr *prefix, int prefixlen,
                                           const struct in6_addr *gwaddr,
                                           struct net_device *dev,
                                           unsigned int pref)
{
        struct fib6_config cfg = {
                .fc_metric        = IP6_RT_PRIO_USER,
                .fc_ifindex        = dev->ifindex,
                .fc_dst_len        = prefixlen,
                .fc_flags        = RTF_GATEWAY | RTF_ADDRCONF | RTF_ROUTEINFO |
                                  RTF_UP | RTF_PREF(pref),
                .fc_protocol = RTPROT_RA,
                .fc_type = RTN_UNICAST,
                .fc_nlinfo.portid = 0,
                .fc_nlinfo.nlh = NULL,
                .fc_nlinfo.nl_net = net,
        };

        cfg.fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO;
        cfg.fc_dst = *prefix;
        cfg.fc_gateway = *gwaddr;

        /* We should treat it as a default route if prefix length is 0. */
        if (!prefixlen)
                cfg.fc_flags |= RTF_DEFAULT;

        ip6_route_add(&cfg, GFP_ATOMIC, NULL);

        return rt6_get_route_info(net, prefix, prefixlen, gwaddr, dev);
}
#endif

struct fib6_info *rt6_get_dflt_router(struct net *net,
                                     const struct in6_addr *addr,
                                     struct net_device *dev)
{
        u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT;
        struct fib6_info *rt;
        struct fib6_table *table;

        table = fib6_get_table(net, tb_id);
        if (!table)
                return NULL;

        rcu_read_lock();
        for_each_fib6_node_rt_rcu(&table->tb6_root) {
                struct fib6_nh *nh;

                /* RA routes do not use nexthops */
                if (rt->nh)
                        continue;

                nh = rt->fib6_nh;
                if (dev == nh->fib_nh_dev &&
                    ((rt->fib6_flags & (RTF_ADDRCONF | RTF_DEFAULT)) == (RTF_ADDRCONF | RTF_DEFAULT)) &&
                    ipv6_addr_equal(&nh->fib_nh_gw6, addr))
                        break;
        }
        if (rt && !fib6_info_hold_safe(rt))
                rt = NULL;
        rcu_read_unlock();
        return rt;
}

struct fib6_info *rt6_add_dflt_router(struct net *net,
                                     const struct in6_addr *gwaddr,
                                     struct net_device *dev,
                                     unsigned int pref,
                                     u32 defrtr_usr_metric,
                                     int lifetime)
{
        struct fib6_config cfg = {
                .fc_table        = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT,
                .fc_metric        = defrtr_usr_metric,
                .fc_ifindex        = dev->ifindex,
                .fc_flags        = RTF_GATEWAY | RTF_ADDRCONF | RTF_DEFAULT |
                                  RTF_UP | RTF_EXPIRES | RTF_PREF(pref),
                .fc_protocol = RTPROT_RA,
                .fc_type = RTN_UNICAST,
                .fc_nlinfo.portid = 0,
                .fc_nlinfo.nlh = NULL,
                .fc_nlinfo.nl_net = net,
                .fc_expires = jiffies_to_clock_t(lifetime * HZ),
        };

        cfg.fc_gateway = *gwaddr;

        if (!ip6_route_add(&cfg, GFP_ATOMIC, NULL)) {
                struct fib6_table *table;

                table = fib6_get_table(dev_net(dev), cfg.fc_table);
                if (table)
                        table->flags |= RT6_TABLE_HAS_DFLT_ROUTER;
        }

        return rt6_get_dflt_router(net, gwaddr, dev);
}

static void __rt6_purge_dflt_routers(struct net *net,
                                     struct fib6_table *table)
{
        struct fib6_info *rt;

restart:
        rcu_read_lock();
        for_each_fib6_node_rt_rcu(&table->tb6_root) {
                struct net_device *dev = fib6_info_nh_dev(rt);
                struct inet6_dev *idev = dev ? __in6_dev_get(dev) : NULL;

                if (rt->fib6_flags & (RTF_DEFAULT | RTF_ADDRCONF) &&
                    (!idev || idev->cnf.accept_ra != 2) &&
                    fib6_info_hold_safe(rt)) {
                        rcu_read_unlock();
                        ip6_del_rt(net, rt, false);
                        goto restart;
                }
        }
        rcu_read_unlock();

        table->flags &= ~RT6_TABLE_HAS_DFLT_ROUTER;
}

void rt6_purge_dflt_routers(struct net *net)
{
        struct fib6_table *table;
        struct hlist_head *head;
        unsigned int h;

        rcu_read_lock();

        for (h = 0; h < FIB6_TABLE_HASHSZ; h++) {
                head = &net->ipv6.fib_table_hash[h];
                hlist_for_each_entry_rcu(table, head, tb6_hlist) {
                        if (table->flags & RT6_TABLE_HAS_DFLT_ROUTER)
                                __rt6_purge_dflt_routers(net, table);
                }
        }

        rcu_read_unlock();
}

static void rtmsg_to_fib6_config(struct net *net,
                                 struct in6_rtmsg *rtmsg,
                                 struct fib6_config *cfg)
{
        *cfg = (struct fib6_config){
                .fc_table = l3mdev_fib_table_by_index(net, rtmsg->rtmsg_ifindex) ?
                         : RT6_TABLE_MAIN,
                .fc_ifindex = rtmsg->rtmsg_ifindex,
                .fc_metric = rtmsg->rtmsg_metric,
                .fc_expires = rtmsg->rtmsg_info,
                .fc_dst_len = rtmsg->rtmsg_dst_len,
                .fc_src_len = rtmsg->rtmsg_src_len,
                .fc_flags = rtmsg->rtmsg_flags,
                .fc_type = rtmsg->rtmsg_type,

                .fc_nlinfo.nl_net = net,

                .fc_dst = rtmsg->rtmsg_dst,
                .fc_src = rtmsg->rtmsg_src,
                .fc_gateway = rtmsg->rtmsg_gateway,
        };
}

int ipv6_route_ioctl(struct net *net, unsigned int cmd, struct in6_rtmsg *rtmsg)
{
        struct fib6_config cfg;
        int err;

        if (cmd != SIOCADDRT && cmd != SIOCDELRT)
                return -EINVAL;
        if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
                return -EPERM;

        rtmsg_to_fib6_config(net, rtmsg, &cfg);

        rtnl_lock();
        switch (cmd) {
        case SIOCADDRT:
                /* Only do the default setting of fc_metric in route adding */
                if (cfg.fc_metric == 0)
                        cfg.fc_metric = IP6_RT_PRIO_USER;
                err = ip6_route_add(&cfg, GFP_KERNEL, NULL);
                break;
        case SIOCDELRT:
                err = ip6_route_del(&cfg, NULL);
                break;
        }
        rtnl_unlock();
        return err;
}

/*
 *        Drop the packet on the floor
 */

static int ip6_pkt_drop(struct sk_buff *skb, u8 code, int ipstats_mib_noroutes)
{
        struct dst_entry *dst = skb_dst(skb);
        struct net *net = dev_net(dst->dev);
        struct inet6_dev *idev;
        SKB_DR(reason);
        int type;

        if (netif_is_l3_master(skb->dev) ||
            dst->dev == net->loopback_dev)
                idev = __in6_dev_get_safely(dev_get_by_index_rcu(net, IP6CB(skb)->iif));
        else
                idev = ip6_dst_idev(dst);

        switch (ipstats_mib_noroutes) {
        case IPSTATS_MIB_INNOROUTES:
                type = ipv6_addr_type(&ipv6_hdr(skb)->daddr);
                if (type == IPV6_ADDR_ANY) {
                        SKB_DR_SET(reason, IP_INADDRERRORS);
                        IP6_INC_STATS(net, idev, IPSTATS_MIB_INADDRERRORS);
                        break;
                }
                SKB_DR_SET(reason, IP_INNOROUTES);
                fallthrough;
        case IPSTATS_MIB_OUTNOROUTES:
                SKB_DR_OR(reason, IP_OUTNOROUTES);
                IP6_INC_STATS(net, idev, ipstats_mib_noroutes);
                break;
        }

        /* Start over by dropping the dst for l3mdev case */
        if (netif_is_l3_master(skb->dev))
                skb_dst_drop(skb);

        icmpv6_send(skb, ICMPV6_DEST_UNREACH, code, 0);
        kfree_skb_reason(skb, reason);
        return 0;
}

static int ip6_pkt_discard(struct sk_buff *skb)
{
        return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_INNOROUTES);
}

static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb)
{
        skb->dev = skb_dst(skb)->dev;
        return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_OUTNOROUTES);
}

static int ip6_pkt_prohibit(struct sk_buff *skb)
{
        return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_INNOROUTES);
}

static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb)
{
        skb->dev = skb_dst(skb)->dev;
        return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_OUTNOROUTES);
}

/*
 *        Allocate a dst for local (unicast / anycast) address.
 */

struct fib6_info *addrconf_f6i_alloc(struct net *net,
                                     struct inet6_dev *idev,
                                     const struct in6_addr *addr,
                                     bool anycast, gfp_t gfp_flags,
                                     struct netlink_ext_ack *extack)
{
        struct fib6_config cfg = {
                .fc_table = l3mdev_fib_table(idev->dev) ? : RT6_TABLE_LOCAL,
                .fc_ifindex = idev->dev->ifindex,
                .fc_flags = RTF_UP | RTF_NONEXTHOP,
                .fc_dst = *addr,
                .fc_dst_len = 128,
                .fc_protocol = RTPROT_KERNEL,
                .fc_nlinfo.nl_net = net,
                .fc_ignore_dev_down = true,
        };
        struct fib6_info *f6i;

        if (anycast) {
                cfg.fc_type = RTN_ANYCAST;
                cfg.fc_flags |= RTF_ANYCAST;
        } else {
                cfg.fc_type = RTN_LOCAL;
                cfg.fc_flags |= RTF_LOCAL;
        }

        f6i = ip6_route_info_create(&cfg, gfp_flags, extack);
        if (!IS_ERR(f6i)) {
                f6i->dst_nocount = true;

                if (!anycast &&
                    (READ_ONCE(net->ipv6.devconf_all->disable_policy) ||
                     READ_ONCE(idev->cnf.disable_policy)))
                        f6i->dst_nopolicy = true;
        }

        return f6i;
}

/* remove deleted ip from prefsrc entries */
struct arg_dev_net_ip {
        struct net *net;
        struct in6_addr *addr;
};

static int fib6_remove_prefsrc(struct fib6_info *rt, void *arg)
{
        struct net *net = ((struct arg_dev_net_ip *)arg)->net;
        struct in6_addr *addr = ((struct arg_dev_net_ip *)arg)->addr;

        if (!rt->nh &&
            rt != net->ipv6.fib6_null_entry &&
            ipv6_addr_equal(addr, &rt->fib6_prefsrc.addr) &&
            !ipv6_chk_addr(net, addr, rt->fib6_nh->fib_nh_dev, 0)) {
                spin_lock_bh(&rt6_exception_lock);
                /* remove prefsrc entry */
                rt->fib6_prefsrc.plen = 0;
                spin_unlock_bh(&rt6_exception_lock);
        }
        return 0;
}

void rt6_remove_prefsrc(struct inet6_ifaddr *ifp)
{
        struct net *net = dev_net(ifp->idev->dev);
        struct arg_dev_net_ip adni = {
                .net = net,
                .addr = &ifp->addr,
        };
        fib6_clean_all(net, fib6_remove_prefsrc, &adni);
}

#define RTF_RA_ROUTER                (RTF_ADDRCONF | RTF_DEFAULT)

/* Remove routers and update dst entries when gateway turn into host. */
static int fib6_clean_tohost(struct fib6_info *rt, void *arg)
{
        struct in6_addr *gateway = (struct in6_addr *)arg;
        struct fib6_nh *nh;

        /* RA routes do not use nexthops */
        if (rt->nh)
                return 0;

        nh = rt->fib6_nh;
        if (((rt->fib6_flags & RTF_RA_ROUTER) == RTF_RA_ROUTER) &&
            nh->fib_nh_gw_family && ipv6_addr_equal(gateway, &nh->fib_nh_gw6))
                return -1;

        /* Further clean up cached routes in exception table.
         * This is needed because cached route may have a different
         * gateway than its 'parent' in the case of an ip redirect.
         */
        fib6_nh_exceptions_clean_tohost(nh, gateway);

        return 0;
}

void rt6_clean_tohost(struct net *net, struct in6_addr *gateway)
{
        fib6_clean_all(net, fib6_clean_tohost, gateway);
}

struct arg_netdev_event {
        const struct net_device *dev;
        union {
                unsigned char nh_flags;
                unsigned long event;
        };
};

static struct fib6_info *rt6_multipath_first_sibling(const struct fib6_info *rt)
{
        struct fib6_info *iter;
        struct fib6_node *fn;

        fn = rcu_dereference_protected(rt->fib6_node,
                        lockdep_is_held(&rt->fib6_table->tb6_lock));
        iter = rcu_dereference_protected(fn->leaf,
                        lockdep_is_held(&rt->fib6_table->tb6_lock));
        while (iter) {
                if (iter->fib6_metric == rt->fib6_metric &&
                    rt6_qualify_for_ecmp(iter))
                        return iter;
                iter = rcu_dereference_protected(iter->fib6_next,
                                lockdep_is_held(&rt->fib6_table->tb6_lock));
        }

        return NULL;
}

/* only called for fib entries with builtin fib6_nh */
static bool rt6_is_dead(const struct fib6_info *rt)
{
        if (rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD ||
            (rt->fib6_nh->fib_nh_flags & RTNH_F_LINKDOWN &&
             ip6_ignore_linkdown(rt->fib6_nh->fib_nh_dev)))
                return true;

        return false;
}

static int rt6_multipath_total_weight(const struct fib6_info *rt)
{
        struct fib6_info *iter;
        int total = 0;

        if (!rt6_is_dead(rt))
                total += rt->fib6_nh->fib_nh_weight;

        list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) {
                if (!rt6_is_dead(iter))
                        total += iter->fib6_nh->fib_nh_weight;
        }

        return total;
}

static void rt6_upper_bound_set(struct fib6_info *rt, int *weight, int total)
{
        int upper_bound = -1;

        if (!rt6_is_dead(rt)) {
                *weight += rt->fib6_nh->fib_nh_weight;
                upper_bound = DIV_ROUND_CLOSEST_ULL((u64) (*weight) << 31,
                                                    total) - 1;
        }
        atomic_set(&rt->fib6_nh->fib_nh_upper_bound, upper_bound);
}

static void rt6_multipath_upper_bound_set(struct fib6_info *rt, int total)
{
        struct fib6_info *iter;
        int weight = 0;

        rt6_upper_bound_set(rt, &weight, total);

        list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
                rt6_upper_bound_set(iter, &weight, total);
}

void rt6_multipath_rebalance(struct fib6_info *rt)
{
        struct fib6_info *first;
        int total;

        /* In case the entire multipath route was marked for flushing,
         * then there is no need to rebalance upon the removal of every
         * sibling route.
         */
        if (!rt->fib6_nsiblings || rt->should_flush)
                return;

        /* During lookup routes are evaluated in order, so we need to
         * make sure upper bounds are assigned from the first sibling
         * onwards.
         */
        first = rt6_multipath_first_sibling(rt);
        if (WARN_ON_ONCE(!first))
                return;

        total = rt6_multipath_total_weight(first);
        rt6_multipath_upper_bound_set(first, total);
}

static int fib6_ifup(struct fib6_info *rt, void *p_arg)
{
        const struct arg_netdev_event *arg = p_arg;
        struct net *net = dev_net(arg->dev);

        if (rt != net->ipv6.fib6_null_entry && !rt->nh &&
            rt->fib6_nh->fib_nh_dev == arg->dev) {
                rt->fib6_nh->fib_nh_flags &= ~arg->nh_flags;
                fib6_update_sernum_upto_root(net, rt);
                rt6_multipath_rebalance(rt);
        }

        return 0;
}

void rt6_sync_up(struct net_device *dev, unsigned char nh_flags)
{
        struct arg_netdev_event arg = {
                .dev = dev,
                {
                        .nh_flags = nh_flags,
                },
        };

        if (nh_flags & RTNH_F_DEAD && netif_carrier_ok(dev))
                arg.nh_flags |= RTNH_F_LINKDOWN;

        fib6_clean_all(dev_net(dev), fib6_ifup, &arg);
}

/* only called for fib entries with inline fib6_nh */
static bool rt6_multipath_uses_dev(const struct fib6_info *rt,
                                   const struct net_device *dev)
{
        struct fib6_info *iter;

        if (rt->fib6_nh->fib_nh_dev == dev)
                return true;
        list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
                if (iter->fib6_nh->fib_nh_dev == dev)
                        return true;

        return false;
}

static void rt6_multipath_flush(struct fib6_info *rt)
{
        struct fib6_info *iter;

        rt->should_flush = 1;
        list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
                iter->should_flush = 1;
}

static unsigned int rt6_multipath_dead_count(const struct fib6_info *rt,
                                             const struct net_device *down_dev)
{
        struct fib6_info *iter;
        unsigned int dead = 0;

        if (rt->fib6_nh->fib_nh_dev == down_dev ||
            rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD)
                dead++;
        list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
                if (iter->fib6_nh->fib_nh_dev == down_dev ||
                    iter->fib6_nh->fib_nh_flags & RTNH_F_DEAD)
                        dead++;

        return dead;
}

static void rt6_multipath_nh_flags_set(struct fib6_info *rt,
                                       const struct net_device *dev,
                                       unsigned char nh_flags)
{
        struct fib6_info *iter;

        if (rt->fib6_nh->fib_nh_dev == dev)
                rt->fib6_nh->fib_nh_flags |= nh_flags;
        list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
                if (iter->fib6_nh->fib_nh_dev == dev)
                        iter->fib6_nh->fib_nh_flags |= nh_flags;
}

/* called with write lock held for table with rt */
static int fib6_ifdown(struct fib6_info *rt, void *p_arg)
{
        const struct arg_netdev_event *arg = p_arg;
        const struct net_device *dev = arg->dev;
        struct net *net = dev_net(dev);

        if (rt == net->ipv6.fib6_null_entry || rt->nh)
                return 0;

        switch (arg->event) {
        case NETDEV_UNREGISTER:
                return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0;
        case NETDEV_DOWN:
                if (rt->should_flush)
                        return -1;
                if (!rt->fib6_nsiblings)
                        return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0;
                if (rt6_multipath_uses_dev(rt, dev)) {
                        unsigned int count;

                        count = rt6_multipath_dead_count(rt, dev);
                        if (rt->fib6_nsiblings + 1 == count) {
                                rt6_multipath_flush(rt);
                                return -1;
                        }
                        rt6_multipath_nh_flags_set(rt, dev, RTNH_F_DEAD |
                                                   RTNH_F_LINKDOWN);
                        fib6_update_sernum(net, rt);
                        rt6_multipath_rebalance(rt);
                }
                return -2;
        case NETDEV_CHANGE:
                if (rt->fib6_nh->fib_nh_dev != dev ||
                    rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST))
                        break;
                rt->fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN;
                rt6_multipath_rebalance(rt);
                break;
        }

        return 0;
}

void rt6_sync_down_dev(struct net_device *dev, unsigned long event)
{
        struct arg_netdev_event arg = {
                .dev = dev,
                {
                        .event = event,
                },
        };
        struct net *net = dev_net(dev);

        if (net->ipv6.sysctl.skip_notify_on_dev_down)
                fib6_clean_all_skip_notify(net, fib6_ifdown, &arg);
        else
                fib6_clean_all(net, fib6_ifdown, &arg);
}

void rt6_disable_ip(struct net_device *dev, unsigned long event)
{
        rt6_sync_down_dev(dev, event);
        rt6_uncached_list_flush_dev(dev);
        neigh_ifdown(&nd_tbl, dev);
}

struct rt6_mtu_change_arg {
        struct net_device *dev;
        unsigned int mtu;
        struct fib6_info *f6i;
};

static int fib6_nh_mtu_change(struct fib6_nh *nh, void *_arg)
{
        struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *)_arg;
        struct fib6_info *f6i = arg->f6i;

        /* For administrative MTU increase, there is no way to discover
         * IPv6 PMTU increase, so PMTU increase should be updated here.
         * Since RFC 1981 doesn't include administrative MTU increase
         * update PMTU increase is a MUST. (i.e. jumbo frame)
         */
        if (nh->fib_nh_dev == arg->dev) {
                struct inet6_dev *idev = __in6_dev_get(arg->dev);
                u32 mtu = f6i->fib6_pmtu;

                if (mtu >= arg->mtu ||
                    (mtu < arg->mtu && mtu == idev->cnf.mtu6))
                        fib6_metric_set(f6i, RTAX_MTU, arg->mtu);

                spin_lock_bh(&rt6_exception_lock);
                rt6_exceptions_update_pmtu(idev, nh, arg->mtu);
                spin_unlock_bh(&rt6_exception_lock);
        }

        return 0;
}

static int rt6_mtu_change_route(struct fib6_info *f6i, void *p_arg)
{
        struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *) p_arg;
        struct inet6_dev *idev;

        /* In IPv6 pmtu discovery is not optional,
           so that RTAX_MTU lock cannot disable it.
           We still use this lock to block changes
           caused by addrconf/ndisc.
        */

        idev = __in6_dev_get(arg->dev);
        if (!idev)
                return 0;

        if (fib6_metric_locked(f6i, RTAX_MTU))
                return 0;

        arg->f6i = f6i;
        if (f6i->nh) {
                /* fib6_nh_mtu_change only returns 0, so this is safe */
                return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_mtu_change,
                                                arg);
        }

        return fib6_nh_mtu_change(f6i->fib6_nh, arg);
}

void rt6_mtu_change(struct net_device *dev, unsigned int mtu)
{
        struct rt6_mtu_change_arg arg = {
                .dev = dev,
                .mtu = mtu,
        };

        fib6_clean_all(dev_net(dev), rt6_mtu_change_route, &arg);
}

static const struct nla_policy rtm_ipv6_policy[RTA_MAX+1] = {
        [RTA_UNSPEC]                = { .strict_start_type = RTA_DPORT + 1 },
        [RTA_GATEWAY]           = { .len = sizeof(struct in6_addr) },
        [RTA_PREFSRC]                = { .len = sizeof(struct in6_addr) },
        [RTA_OIF]               = { .type = NLA_U32 },
        [RTA_IIF]                = { .type = NLA_U32 },
        [RTA_PRIORITY]          = { .type = NLA_U32 },
        [RTA_METRICS]           = { .type = NLA_NESTED },
        [RTA_MULTIPATH]                = { .len = sizeof(struct rtnexthop) },
        [RTA_PREF]              = { .type = NLA_U8 },
        [RTA_ENCAP_TYPE]        = { .type = NLA_U16 },
        [RTA_ENCAP]                = { .type = NLA_NESTED },
        [RTA_EXPIRES]                = { .type = NLA_U32 },
        [RTA_UID]                = { .type = NLA_U32 },
        [RTA_MARK]                = { .type = NLA_U32 },
        [RTA_TABLE]                = { .type = NLA_U32 },
        [RTA_IP_PROTO]                = { .type = NLA_U8 },
        [RTA_SPORT]                = { .type = NLA_U16 },
        [RTA_DPORT]                = { .type = NLA_U16 },
        [RTA_NH_ID]                = { .type = NLA_U32 },
        [RTA_FLOWLABEL]                = { .type = NLA_BE32 },
};

static int rtm_to_fib6_config(struct sk_buff *skb, struct nlmsghdr *nlh,
                              struct fib6_config *cfg,
                              struct netlink_ext_ack *extack)
{
        struct rtmsg *rtm;
        struct nlattr *tb[RTA_MAX+1];
        unsigned int pref;
        int err;

        err = nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX,
                                     rtm_ipv6_policy, extack);
        if (err < 0)
                goto errout;

        err = -EINVAL;
        rtm = nlmsg_data(nlh);

        if (rtm->rtm_tos) {
                NL_SET_ERR_MSG(extack,
                               "Invalid dsfield (tos): option not available for IPv6");
                goto errout;
        }

        if (tb[RTA_FLOWLABEL]) {
                NL_SET_ERR_MSG_ATTR(extack, tb[RTA_FLOWLABEL],
                                    "Flow label cannot be specified for this operation");
                goto errout;
        }

        *cfg = (struct fib6_config){
                .fc_table = rtm->rtm_table,
                .fc_dst_len = rtm->rtm_dst_len,
                .fc_src_len = rtm->rtm_src_len,
                .fc_flags = RTF_UP,
                .fc_protocol = rtm->rtm_protocol,
                .fc_type = rtm->rtm_type,

                .fc_nlinfo.portid = NETLINK_CB(skb).portid,
                .fc_nlinfo.nlh = nlh,
                .fc_nlinfo.nl_net = sock_net(skb->sk),
        };

        if (rtm->rtm_type == RTN_UNREACHABLE ||
            rtm->rtm_type == RTN_BLACKHOLE ||
            rtm->rtm_type == RTN_PROHIBIT ||
            rtm->rtm_type == RTN_THROW)
                cfg->fc_flags |= RTF_REJECT;

        if (rtm->rtm_type == RTN_LOCAL)
                cfg->fc_flags |= RTF_LOCAL;

        if (rtm->rtm_flags & RTM_F_CLONED)
                cfg->fc_flags |= RTF_CACHE;

        cfg->fc_flags |= (rtm->rtm_flags & RTNH_F_ONLINK);

        if (tb[RTA_NH_ID]) {
                if (tb[RTA_GATEWAY]   || tb[RTA_OIF] ||
                    tb[RTA_MULTIPATH] || tb[RTA_ENCAP]) {
                        NL_SET_ERR_MSG(extack,
                                       "Nexthop specification and nexthop id are mutually exclusive");
                        goto errout;
                }
                cfg->fc_nh_id = nla_get_u32(tb[RTA_NH_ID]);
        }

        if (tb[RTA_GATEWAY]) {
                cfg->fc_gateway = nla_get_in6_addr(tb[RTA_GATEWAY]);
                cfg->fc_flags |= RTF_GATEWAY;
        }
        if (tb[RTA_VIA]) {
                NL_SET_ERR_MSG(extack, "IPv6 does not support RTA_VIA attribute");
                goto errout;
        }

        if (tb[RTA_DST]) {
                int plen = (rtm->rtm_dst_len + 7) >> 3;

                if (nla_len(tb[RTA_DST]) < plen)
                        goto errout;

                nla_memcpy(&cfg->fc_dst, tb[RTA_DST], plen);
        }

        if (tb[RTA_SRC]) {
                int plen = (rtm->rtm_src_len + 7) >> 3;

                if (nla_len(tb[RTA_SRC]) < plen)
                        goto errout;

                nla_memcpy(&cfg->fc_src, tb[RTA_SRC], plen);
        }

        if (tb[RTA_PREFSRC])
                cfg->fc_prefsrc = nla_get_in6_addr(tb[RTA_PREFSRC]);

        if (tb[RTA_OIF])
                cfg->fc_ifindex = nla_get_u32(tb[RTA_OIF]);

        if (tb[RTA_PRIORITY])
                cfg->fc_metric = nla_get_u32(tb[RTA_PRIORITY]);

        if (tb[RTA_METRICS]) {
                cfg->fc_mx = nla_data(tb[RTA_METRICS]);
                cfg->fc_mx_len = nla_len(tb[RTA_METRICS]);
        }

        if (tb[RTA_TABLE])
                cfg->fc_table = nla_get_u32(tb[RTA_TABLE]);

        if (tb[RTA_MULTIPATH]) {
                cfg->fc_mp = nla_data(tb[RTA_MULTIPATH]);
                cfg->fc_mp_len = nla_len(tb[RTA_MULTIPATH]);

                err = lwtunnel_valid_encap_type_attr(cfg->fc_mp,
                                                     cfg->fc_mp_len,
                                                     extack, true);
                if (err < 0)
                        goto errout;
        }

        if (tb[RTA_PREF]) {
                pref = nla_get_u8(tb[RTA_PREF]);
                if (pref != ICMPV6_ROUTER_PREF_LOW &&
                    pref != ICMPV6_ROUTER_PREF_HIGH)
                        pref = ICMPV6_ROUTER_PREF_MEDIUM;
                cfg->fc_flags |= RTF_PREF(pref);
        }

        if (tb[RTA_ENCAP])
                cfg->fc_encap = tb[RTA_ENCAP];

        if (tb[RTA_ENCAP_TYPE]) {
                cfg->fc_encap_type = nla_get_u16(tb[RTA_ENCAP_TYPE]);

                err = lwtunnel_valid_encap_type(cfg->fc_encap_type,
                                                extack, true);
                if (err < 0)
                        goto errout;
        }

        if (tb[RTA_EXPIRES]) {
                unsigned long timeout = addrconf_timeout_fixup(nla_get_u32(tb[RTA_EXPIRES]), HZ);

                if (addrconf_finite_timeout(timeout)) {
                        cfg->fc_expires = jiffies_to_clock_t(timeout * HZ);
                        cfg->fc_flags |= RTF_EXPIRES;
                }
        }

        err = 0;
errout:
        return err;
}

struct rt6_nh {
        struct fib6_info *fib6_info;
        struct fib6_config r_cfg;
        struct list_head next;
};

static int ip6_route_info_append(struct net *net,
                                 struct list_head *rt6_nh_list,
                                 struct fib6_info *rt,
                                 struct fib6_config *r_cfg)
{
        struct rt6_nh *nh;
        int err = -EEXIST;

        list_for_each_entry(nh, rt6_nh_list, next) {
                /* check if fib6_info already exists */
                if (rt6_duplicate_nexthop(nh->fib6_info, rt))
                        return err;
        }

        nh = kzalloc(sizeof(*nh), GFP_KERNEL);
        if (!nh)
                return -ENOMEM;
        nh->fib6_info = rt;
        memcpy(&nh->r_cfg, r_cfg, sizeof(*r_cfg));
        list_add_tail(&nh->next, rt6_nh_list);

        return 0;
}

static void ip6_route_mpath_notify(struct fib6_info *rt,
                                   struct fib6_info *rt_last,
                                   struct nl_info *info,
                                   __u16 nlflags)
{
        /* if this is an APPEND route, then rt points to the first route
         * inserted and rt_last points to last route inserted. Userspace
         * wants a consistent dump of the route which starts at the first
         * nexthop. Since sibling routes are always added at the end of
         * the list, find the first sibling of the last route appended
         */
        rcu_read_lock();

        if ((nlflags & NLM_F_APPEND) && rt_last && rt_last->fib6_nsiblings) {
                rt = list_first_or_null_rcu(&rt_last->fib6_siblings,
                                            struct fib6_info,
                                            fib6_siblings);
        }

        if (rt)
                inet6_rt_notify(RTM_NEWROUTE, rt, info, nlflags);

        rcu_read_unlock();
}

static bool ip6_route_mpath_should_notify(const struct fib6_info *rt)
{
        bool rt_can_ecmp = rt6_qualify_for_ecmp(rt);
        bool should_notify = false;
        struct fib6_info *leaf;
        struct fib6_node *fn;

        rcu_read_lock();
        fn = rcu_dereference(rt->fib6_node);
        if (!fn)
                goto out;

        leaf = rcu_dereference(fn->leaf);
        if (!leaf)
                goto out;

        if (rt == leaf ||
            (rt_can_ecmp && rt->fib6_metric == leaf->fib6_metric &&
             rt6_qualify_for_ecmp(leaf)))
                should_notify = true;
out:
        rcu_read_unlock();

        return should_notify;
}

static int fib6_gw_from_attr(struct in6_addr *gw, struct nlattr *nla,
                             struct netlink_ext_ack *extack)
{
        if (nla_len(nla) < sizeof(*gw)) {
                NL_SET_ERR_MSG(extack, "Invalid IPv6 address in RTA_GATEWAY");
                return -EINVAL;
        }

        *gw = nla_get_in6_addr(nla);

        return 0;
}

static int ip6_route_multipath_add(struct fib6_config *cfg,
                                   struct netlink_ext_ack *extack)
{
        struct fib6_info *rt_notif = NULL, *rt_last = NULL;
        struct nl_info *info = &cfg->fc_nlinfo;
        struct fib6_config r_cfg;
        struct rtnexthop *rtnh;
        struct fib6_info *rt;
        struct rt6_nh *err_nh;
        struct rt6_nh *nh, *nh_safe;
        __u16 nlflags;
        int remaining;
        int attrlen;
        int err = 1;
        int nhn = 0;
        int replace = (cfg->fc_nlinfo.nlh &&
                       (cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_REPLACE));
        LIST_HEAD(rt6_nh_list);

        nlflags = replace ? NLM_F_REPLACE : NLM_F_CREATE;
        if (info->nlh && info->nlh->nlmsg_flags & NLM_F_APPEND)
                nlflags |= NLM_F_APPEND;

        remaining = cfg->fc_mp_len;
        rtnh = (struct rtnexthop *)cfg->fc_mp;

        /* Parse a Multipath Entry and build a list (rt6_nh_list) of
         * fib6_info structs per nexthop
         */
        while (rtnh_ok(rtnh, remaining)) {
                memcpy(&r_cfg, cfg, sizeof(*cfg));
                if (rtnh->rtnh_ifindex)
                        r_cfg.fc_ifindex = rtnh->rtnh_ifindex;

                attrlen = rtnh_attrlen(rtnh);
                if (attrlen > 0) {
                        struct nlattr *nla, *attrs = rtnh_attrs(rtnh);

                        nla = nla_find(attrs, attrlen, RTA_GATEWAY);
                        if (nla) {
                                err = fib6_gw_from_attr(&r_cfg.fc_gateway, nla,
                                                        extack);
                                if (err)
                                        goto cleanup;

                                r_cfg.fc_flags |= RTF_GATEWAY;
                        }
                        r_cfg.fc_encap = nla_find(attrs, attrlen, RTA_ENCAP);

                        /* RTA_ENCAP_TYPE length checked in
                         * lwtunnel_valid_encap_type_attr
                         */
                        nla = nla_find(attrs, attrlen, RTA_ENCAP_TYPE);
                        if (nla)
                                r_cfg.fc_encap_type = nla_get_u16(nla);
                }

                r_cfg.fc_flags |= (rtnh->rtnh_flags & RTNH_F_ONLINK);
                rt = ip6_route_info_create(&r_cfg, GFP_KERNEL, extack);
                if (IS_ERR(rt)) {
                        err = PTR_ERR(rt);
                        rt = NULL;
                        goto cleanup;
                }
                if (!rt6_qualify_for_ecmp(rt)) {
                        err = -EINVAL;
                        NL_SET_ERR_MSG(extack,
                                       "Device only routes can not be added for IPv6 using the multipath API.");
                        fib6_info_release(rt);
                        goto cleanup;
                }

                rt->fib6_nh->fib_nh_weight = rtnh->rtnh_hops + 1;

                err = ip6_route_info_append(info->nl_net, &rt6_nh_list,
                                            rt, &r_cfg);
                if (err) {
                        fib6_info_release(rt);
                        goto cleanup;
                }

                rtnh = rtnh_next(rtnh, &remaining);
        }

        if (list_empty(&rt6_nh_list)) {
                NL_SET_ERR_MSG(extack,
                               "Invalid nexthop configuration - no valid nexthops");
                return -EINVAL;
        }

        /* for add and replace send one notification with all nexthops.
         * Skip the notification in fib6_add_rt2node and send one with
         * the full route when done
         */
        info->skip_notify = 1;

        /* For add and replace, send one notification with all nexthops. For
         * append, send one notification with all appended nexthops.
         */
        info->skip_notify_kernel = 1;

        err_nh = NULL;
        list_for_each_entry(nh, &rt6_nh_list, next) {
                err = __ip6_ins_rt(nh->fib6_info, info, extack);

                if (err) {
                        if (replace && nhn)
                                NL_SET_ERR_MSG_MOD(extack,
                                                   "multipath route replace failed (check consistency of installed routes)");
                        err_nh = nh;
                        goto add_errout;
                }
                /* save reference to last route successfully inserted */
                rt_last = nh->fib6_info;

                /* save reference to first route for notification */
                if (!rt_notif)
                        rt_notif = nh->fib6_info;

                /* Because each route is added like a single route we remove
                 * these flags after the first nexthop: if there is a collision,
                 * we have already failed to add the first nexthop:
                 * fib6_add_rt2node() has rejected it; when replacing, old
                 * nexthops have been replaced by first new, the rest should
                 * be added to it.
                 */
                if (cfg->fc_nlinfo.nlh) {
                        cfg->fc_nlinfo.nlh->nlmsg_flags &= ~(NLM_F_EXCL |
                                                             NLM_F_REPLACE);
                        cfg->fc_nlinfo.nlh->nlmsg_flags |= NLM_F_CREATE;
                }
                nhn++;
        }

        /* An in-kernel notification should only be sent in case the new
         * multipath route is added as the first route in the node, or if
         * it was appended to it. We pass 'rt_notif' since it is the first
         * sibling and might allow us to skip some checks in the replace case.
         */
        if (ip6_route_mpath_should_notify(rt_notif)) {
                enum fib_event_type fib_event;

                if (rt_notif->fib6_nsiblings != nhn - 1)
                        fib_event = FIB_EVENT_ENTRY_APPEND;
                else
                        fib_event = FIB_EVENT_ENTRY_REPLACE;

                err = call_fib6_multipath_entry_notifiers(info->nl_net,
                                                          fib_event, rt_notif,
                                                          nhn - 1, extack);
                if (err) {
                        /* Delete all the siblings that were just added */
                        err_nh = NULL;
                        goto add_errout;
                }
        }

        /* success ... tell user about new route */
        ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags);
        goto cleanup;

add_errout:
        /* send notification for routes that were added so that
         * the delete notifications sent by ip6_route_del are
         * coherent
         */
        if (rt_notif)
                ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags);

        /* Delete routes that were already added */
        list_for_each_entry(nh, &rt6_nh_list, next) {
                if (err_nh == nh)
                        break;
                ip6_route_del(&nh->r_cfg, extack);
        }

cleanup:
        list_for_each_entry_safe(nh, nh_safe, &rt6_nh_list, next) {
                fib6_info_release(nh->fib6_info);
                list_del(&nh->next);
                kfree(nh);
        }

        return err;
}

static int ip6_route_multipath_del(struct fib6_config *cfg,
                                   struct netlink_ext_ack *extack)
{
        struct fib6_config r_cfg;
        struct rtnexthop *rtnh;
        int last_err = 0;
        int remaining;
        int attrlen;
        int err;

        remaining = cfg->fc_mp_len;
        rtnh = (struct rtnexthop *)cfg->fc_mp;

        /* Parse a Multipath Entry */
        while (rtnh_ok(rtnh, remaining)) {
                memcpy(&r_cfg, cfg, sizeof(*cfg));
                if (rtnh->rtnh_ifindex)
                        r_cfg.fc_ifindex = rtnh->rtnh_ifindex;

                attrlen = rtnh_attrlen(rtnh);
                if (attrlen > 0) {
                        struct nlattr *nla, *attrs = rtnh_attrs(rtnh);

                        nla = nla_find(attrs, attrlen, RTA_GATEWAY);
                        if (nla) {
                                err = fib6_gw_from_attr(&r_cfg.fc_gateway, nla,
                                                        extack);
                                if (err) {
                                        last_err = err;
                                        goto next_rtnh;
                                }

                                r_cfg.fc_flags |= RTF_GATEWAY;
                        }
                }
                err = ip6_route_del(&r_cfg, extack);
                if (err)
                        last_err = err;

next_rtnh:
                rtnh = rtnh_next(rtnh, &remaining);
        }

        return last_err;
}

static int inet6_rtm_delroute(struct sk_buff *skb, struct nlmsghdr *nlh,
                              struct netlink_ext_ack *extack)
{
        struct fib6_config cfg;
        int err;

        err = rtm_to_fib6_config(skb, nlh, &cfg, extack);
        if (err < 0)
                return err;

        if (cfg.fc_nh_id &&
            !nexthop_find_by_id(sock_net(skb->sk), cfg.fc_nh_id)) {
                NL_SET_ERR_MSG(extack, "Nexthop id does not exist");
                return -EINVAL;
        }

        if (cfg.fc_mp)
                return ip6_route_multipath_del(&cfg, extack);
        else {
                cfg.fc_delete_all_nh = 1;
                return ip6_route_del(&cfg, extack);
        }
}

static int inet6_rtm_newroute(struct sk_buff *skb, struct nlmsghdr *nlh,
                              struct netlink_ext_ack *extack)
{
        struct fib6_config cfg;
        int err;

        err = rtm_to_fib6_config(skb, nlh, &cfg, extack);
        if (err < 0)
                return err;

        if (cfg.fc_metric == 0)
                cfg.fc_metric = IP6_RT_PRIO_USER;

        if (cfg.fc_mp)
                return ip6_route_multipath_add(&cfg, extack);
        else
                return ip6_route_add(&cfg, GFP_KERNEL, extack);
}

/* add the overhead of this fib6_nh to nexthop_len */
static int rt6_nh_nlmsg_size(struct fib6_nh *nh, void *arg)
{
        int *nexthop_len = arg;

        *nexthop_len += nla_total_size(0)         /* RTA_MULTIPATH */
                     + NLA_ALIGN(sizeof(struct rtnexthop))
                     + nla_total_size(16); /* RTA_GATEWAY */

        if (nh->fib_nh_lws) {
                /* RTA_ENCAP_TYPE */
                *nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws);
                /* RTA_ENCAP */
                *nexthop_len += nla_total_size(2);
        }

        return 0;
}

static size_t rt6_nlmsg_size(struct fib6_info *f6i)
{
        int nexthop_len;

        if (f6i->nh) {
                nexthop_len = nla_total_size(4); /* RTA_NH_ID */
                nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_nlmsg_size,
                                         &nexthop_len);
        } else {
                struct fib6_nh *nh = f6i->fib6_nh;
                struct fib6_info *sibling;

                nexthop_len = 0;
                if (f6i->fib6_nsiblings) {
                        rt6_nh_nlmsg_size(nh, &nexthop_len);

                        rcu_read_lock();

                        list_for_each_entry_rcu(sibling, &f6i->fib6_siblings,
                                                fib6_siblings) {
                                rt6_nh_nlmsg_size(sibling->fib6_nh, &nexthop_len);
                        }

                        rcu_read_unlock();
                }
                nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws);
        }

        return NLMSG_ALIGN(sizeof(struct rtmsg))
               + nla_total_size(16) /* RTA_SRC */
               + nla_total_size(16) /* RTA_DST */
               + nla_total_size(16) /* RTA_GATEWAY */
               + nla_total_size(16) /* RTA_PREFSRC */
               + nla_total_size(4) /* RTA_TABLE */
               + nla_total_size(4) /* RTA_IIF */
               + nla_total_size(4) /* RTA_OIF */
               + nla_total_size(4) /* RTA_PRIORITY */
               + RTAX_MAX * nla_total_size(4) /* RTA_METRICS */
               + nla_total_size(sizeof(struct rta_cacheinfo))
               + nla_total_size(TCP_CA_NAME_MAX) /* RTAX_CC_ALGO */
               + nla_total_size(1) /* RTA_PREF */
               + nexthop_len;
}

static int rt6_fill_node_nexthop(struct sk_buff *skb, struct nexthop *nh,
                                 unsigned char *flags)
{
        if (nexthop_is_multipath(nh)) {
                struct nlattr *mp;

                mp = nla_nest_start_noflag(skb, RTA_MULTIPATH);
                if (!mp)
                        goto nla_put_failure;

                if (nexthop_mpath_fill_node(skb, nh, AF_INET6))
                        goto nla_put_failure;

                nla_nest_end(skb, mp);
        } else {
                struct fib6_nh *fib6_nh;

                fib6_nh = nexthop_fib6_nh(nh);
                if (fib_nexthop_info(skb, &fib6_nh->nh_common, AF_INET6,
                                     flags, false) < 0)
                        goto nla_put_failure;
        }

        return 0;

nla_put_failure:
        return -EMSGSIZE;
}

static int rt6_fill_node(struct net *net, struct sk_buff *skb,
                         struct fib6_info *rt, struct dst_entry *dst,
                         struct in6_addr *dest, struct in6_addr *src,
                         int iif, int type, u32 portid, u32 seq,
                         unsigned int flags)
{
        struct rt6_info *rt6 = dst_rt6_info(dst);
        struct rt6key *rt6_dst, *rt6_src;
        u32 *pmetrics, table, rt6_flags;
        unsigned char nh_flags = 0;
        struct nlmsghdr *nlh;
        struct rtmsg *rtm;
        long expires = 0;

        nlh = nlmsg_put(skb, portid, seq, type, sizeof(*rtm), flags);
        if (!nlh)
                return -EMSGSIZE;

        if (rt6) {
                rt6_dst = &rt6->rt6i_dst;
                rt6_src = &rt6->rt6i_src;
                rt6_flags = rt6->rt6i_flags;
        } else {
                rt6_dst = &rt->fib6_dst;
                rt6_src = &rt->fib6_src;
                rt6_flags = rt->fib6_flags;
        }

        rtm = nlmsg_data(nlh);
        rtm->rtm_family = AF_INET6;
        rtm->rtm_dst_len = rt6_dst->plen;
        rtm->rtm_src_len = rt6_src->plen;
        rtm->rtm_tos = 0;
        if (rt->fib6_table)
                table = rt->fib6_table->tb6_id;
        else
                table = RT6_TABLE_UNSPEC;
        rtm->rtm_table = table < 256 ? table : RT_TABLE_COMPAT;
        if (nla_put_u32(skb, RTA_TABLE, table))
                goto nla_put_failure;

        rtm->rtm_type = rt->fib6_type;
        rtm->rtm_flags = 0;
        rtm->rtm_scope = RT_SCOPE_UNIVERSE;
        rtm->rtm_protocol = rt->fib6_protocol;

        if (rt6_flags & RTF_CACHE)
                rtm->rtm_flags |= RTM_F_CLONED;

        if (dest) {
                if (nla_put_in6_addr(skb, RTA_DST, dest))
                        goto nla_put_failure;
                rtm->rtm_dst_len = 128;
        } else if (rtm->rtm_dst_len)
                if (nla_put_in6_addr(skb, RTA_DST, &rt6_dst->addr))
                        goto nla_put_failure;
#ifdef CONFIG_IPV6_SUBTREES
        if (src) {
                if (nla_put_in6_addr(skb, RTA_SRC, src))
                        goto nla_put_failure;
                rtm->rtm_src_len = 128;
        } else if (rtm->rtm_src_len &&
                   nla_put_in6_addr(skb, RTA_SRC, &rt6_src->addr))
                goto nla_put_failure;
#endif
        if (iif) {
#ifdef CONFIG_IPV6_MROUTE
                if (ipv6_addr_is_multicast(&rt6_dst->addr)) {
                        int err = ip6mr_get_route(net, skb, rtm, portid);

                        if (err == 0)
                                return 0;
                        if (err < 0)
                                goto nla_put_failure;
                } else
#endif
                        if (nla_put_u32(skb, RTA_IIF, iif))
                                goto nla_put_failure;
        } else if (dest) {
                struct in6_addr saddr_buf;
                if (ip6_route_get_saddr(net, rt, dest, 0, 0, &saddr_buf) == 0 &&
                    nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf))
                        goto nla_put_failure;
        }

        if (rt->fib6_prefsrc.plen) {
                struct in6_addr saddr_buf;
                saddr_buf = rt->fib6_prefsrc.addr;
                if (nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf))
                        goto nla_put_failure;
        }

        pmetrics = dst ? dst_metrics_ptr(dst) : rt->fib6_metrics->metrics;
        if (rtnetlink_put_metrics(skb, pmetrics) < 0)
                goto nla_put_failure;

        if (nla_put_u32(skb, RTA_PRIORITY, rt->fib6_metric))
                goto nla_put_failure;

        /* For multipath routes, walk the siblings list and add
         * each as a nexthop within RTA_MULTIPATH.
         */
        if (rt6) {
                if (rt6_flags & RTF_GATEWAY &&
                    nla_put_in6_addr(skb, RTA_GATEWAY, &rt6->rt6i_gateway))
                        goto nla_put_failure;

                if (dst->dev && nla_put_u32(skb, RTA_OIF, dst->dev->ifindex))
                        goto nla_put_failure;

                if (dst->lwtstate &&
                    lwtunnel_fill_encap(skb, dst->lwtstate, RTA_ENCAP, RTA_ENCAP_TYPE) < 0)
                        goto nla_put_failure;
        } else if (rt->fib6_nsiblings) {
                struct fib6_info *sibling;
                struct nlattr *mp;

                mp = nla_nest_start_noflag(skb, RTA_MULTIPATH);
                if (!mp)
                        goto nla_put_failure;

                if (fib_add_nexthop(skb, &rt->fib6_nh->nh_common,
                                    rt->fib6_nh->fib_nh_weight, AF_INET6,
                                    0) < 0)
                        goto nla_put_failure;

                rcu_read_lock();

                list_for_each_entry_rcu(sibling, &rt->fib6_siblings,
                                        fib6_siblings) {
                        if (fib_add_nexthop(skb, &sibling->fib6_nh->nh_common,
                                            sibling->fib6_nh->fib_nh_weight,
                                            AF_INET6, 0) < 0) {
                                rcu_read_unlock();

                                goto nla_put_failure;
                        }
                }

                rcu_read_unlock();

                nla_nest_end(skb, mp);
        } else if (rt->nh) {
                if (nla_put_u32(skb, RTA_NH_ID, rt->nh->id))
                        goto nla_put_failure;

                if (nexthop_is_blackhole(rt->nh))
                        rtm->rtm_type = RTN_BLACKHOLE;

                if (READ_ONCE(net->ipv4.sysctl_nexthop_compat_mode) &&
                    rt6_fill_node_nexthop(skb, rt->nh, &nh_flags) < 0)
                        goto nla_put_failure;

                rtm->rtm_flags |= nh_flags;
        } else {
                if (fib_nexthop_info(skb, &rt->fib6_nh->nh_common, AF_INET6,
                                     &nh_flags, false) < 0)
                        goto nla_put_failure;

                rtm->rtm_flags |= nh_flags;
        }

        if (rt6_flags & RTF_EXPIRES) {
                expires = dst ? dst->expires : rt->expires;
                expires -= jiffies;
        }

        if (!dst) {
                if (READ_ONCE(rt->offload))
                        rtm->rtm_flags |= RTM_F_OFFLOAD;
                if (READ_ONCE(rt->trap))
                        rtm->rtm_flags |= RTM_F_TRAP;
                if (READ_ONCE(rt->offload_failed))
                        rtm->rtm_flags |= RTM_F_OFFLOAD_FAILED;
        }

        if (rtnl_put_cacheinfo(skb, dst, 0, expires, dst ? dst->error : 0) < 0)
                goto nla_put_failure;

        if (nla_put_u8(skb, RTA_PREF, IPV6_EXTRACT_PREF(rt6_flags)))
                goto nla_put_failure;


        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static int fib6_info_nh_uses_dev(struct fib6_nh *nh, void *arg)
{
        const struct net_device *dev = arg;

        if (nh->fib_nh_dev == dev)
                return 1;

        return 0;
}

static bool fib6_info_uses_dev(const struct fib6_info *f6i,
                               const struct net_device *dev)
{
        if (f6i->nh) {
                struct net_device *_dev = (struct net_device *)dev;

                return !!nexthop_for_each_fib6_nh(f6i->nh,
                                                  fib6_info_nh_uses_dev,
                                                  _dev);
        }

        if (f6i->fib6_nh->fib_nh_dev == dev)
                return true;

        if (f6i->fib6_nsiblings) {
                struct fib6_info *sibling, *next_sibling;

                list_for_each_entry_safe(sibling, next_sibling,
                                         &f6i->fib6_siblings, fib6_siblings) {
                        if (sibling->fib6_nh->fib_nh_dev == dev)
                                return true;
                }
        }

        return false;
}

struct fib6_nh_exception_dump_walker {
        struct rt6_rtnl_dump_arg *dump;
        struct fib6_info *rt;
        unsigned int flags;
        unsigned int skip;
        unsigned int count;
};

static int rt6_nh_dump_exceptions(struct fib6_nh *nh, void *arg)
{
        struct fib6_nh_exception_dump_walker *w = arg;
        struct rt6_rtnl_dump_arg *dump = w->dump;
        struct rt6_exception_bucket *bucket;
        struct rt6_exception *rt6_ex;
        int i, err;

        bucket = fib6_nh_get_excptn_bucket(nh, NULL);
        if (!bucket)
                return 0;

        for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
                hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) {
                        if (w->skip) {
                                w->skip--;
                                continue;
                        }

                        /* Expiration of entries doesn't bump sernum, insertion
                         * does. Removal is triggered by insertion, so we can
                         * rely on the fact that if entries change between two
                         * partial dumps, this node is scanned again completely,
                         * see rt6_insert_exception() and fib6_dump_table().
                         *
                         * Count expired entries we go through as handled
                         * entries that we'll skip next time, in case of partial
                         * node dump. Otherwise, if entries expire meanwhile,
                         * we'll skip the wrong amount.
                         */
                        if (rt6_check_expired(rt6_ex->rt6i)) {
                                w->count++;
                                continue;
                        }

                        err = rt6_fill_node(dump->net, dump->skb, w->rt,
                                            &rt6_ex->rt6i->dst, NULL, NULL, 0,
                                            RTM_NEWROUTE,
                                            NETLINK_CB(dump->cb->skb).portid,
                                            dump->cb->nlh->nlmsg_seq, w->flags);
                        if (err)
                                return err;

                        w->count++;
                }
                bucket++;
        }

        return 0;
}

/* Return -1 if done with node, number of handled routes on partial dump */
int rt6_dump_route(struct fib6_info *rt, void *p_arg, unsigned int skip)
{
        struct rt6_rtnl_dump_arg *arg = (struct rt6_rtnl_dump_arg *) p_arg;
        struct fib_dump_filter *filter = &arg->filter;
        unsigned int flags = NLM_F_MULTI;
        struct net *net = arg->net;
        int count = 0;

        if (rt == net->ipv6.fib6_null_entry)
                return -1;

        if ((filter->flags & RTM_F_PREFIX) &&
            !(rt->fib6_flags & RTF_PREFIX_RT)) {
                /* success since this is not a prefix route */
                return -1;
        }
        if (filter->filter_set &&
            ((filter->rt_type  && rt->fib6_type != filter->rt_type) ||
             (filter->dev      && !fib6_info_uses_dev(rt, filter->dev)) ||
             (filter->protocol && rt->fib6_protocol != filter->protocol))) {
                return -1;
        }

        if (filter->filter_set ||
            !filter->dump_routes || !filter->dump_exceptions) {
                flags |= NLM_F_DUMP_FILTERED;
        }

        if (filter->dump_routes) {
                if (skip) {
                        skip--;
                } else {
                        if (rt6_fill_node(net, arg->skb, rt, NULL, NULL, NULL,
                                          0, RTM_NEWROUTE,
                                          NETLINK_CB(arg->cb->skb).portid,
                                          arg->cb->nlh->nlmsg_seq, flags)) {
                                return 0;
                        }
                        count++;
                }
        }

        if (filter->dump_exceptions) {
                struct fib6_nh_exception_dump_walker w = { .dump = arg,
                                                           .rt = rt,
                                                           .flags = flags,
                                                           .skip = skip,
                                                           .count = 0 };
                int err;

                rcu_read_lock();
                if (rt->nh) {
                        err = nexthop_for_each_fib6_nh(rt->nh,
                                                       rt6_nh_dump_exceptions,
                                                       &w);
                } else {
                        err = rt6_nh_dump_exceptions(rt->fib6_nh, &w);
                }
                rcu_read_unlock();

                if (err)
                        return count + w.count;
        }

        return -1;
}

static int inet6_rtm_valid_getroute_req(struct sk_buff *skb,
                                        const struct nlmsghdr *nlh,
                                        struct nlattr **tb,
                                        struct netlink_ext_ack *extack)
{
        struct rtmsg *rtm;
        int i, err;

        if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*rtm))) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Invalid header for get route request");
                return -EINVAL;
        }

        if (!netlink_strict_get_check(skb))
                return nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX,
                                              rtm_ipv6_policy, extack);

        rtm = nlmsg_data(nlh);
        if ((rtm->rtm_src_len && rtm->rtm_src_len != 128) ||
            (rtm->rtm_dst_len && rtm->rtm_dst_len != 128) ||
            rtm->rtm_table || rtm->rtm_protocol || rtm->rtm_scope ||
            rtm->rtm_type) {
                NL_SET_ERR_MSG_MOD(extack, "Invalid values in header for get route request");
                return -EINVAL;
        }
        if (rtm->rtm_flags & ~RTM_F_FIB_MATCH) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Invalid flags for get route request");
                return -EINVAL;
        }

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(*rtm), tb, RTA_MAX,
                                            rtm_ipv6_policy, extack);
        if (err)
                return err;

        if ((tb[RTA_SRC] && !rtm->rtm_src_len) ||
            (tb[RTA_DST] && !rtm->rtm_dst_len)) {
                NL_SET_ERR_MSG_MOD(extack, "rtm_src_len and rtm_dst_len must be 128 for IPv6");
                return -EINVAL;
        }

        if (tb[RTA_FLOWLABEL] &&
            (nla_get_be32(tb[RTA_FLOWLABEL]) & ~IPV6_FLOWLABEL_MASK)) {
                NL_SET_ERR_MSG_ATTR(extack, tb[RTA_FLOWLABEL],
                                    "Invalid flow label");
                return -EINVAL;
        }

        for (i = 0; i <= RTA_MAX; i++) {
                if (!tb[i])
                        continue;

                switch (i) {
                case RTA_SRC:
                case RTA_DST:
                case RTA_IIF:
                case RTA_OIF:
                case RTA_MARK:
                case RTA_UID:
                case RTA_SPORT:
                case RTA_DPORT:
                case RTA_IP_PROTO:
                case RTA_FLOWLABEL:
                        break;
                default:
                        NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in get route request");
                        return -EINVAL;
                }
        }

        return 0;
}

static int inet6_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
                              struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(in_skb->sk);
        struct nlattr *tb[RTA_MAX+1];
        int err, iif = 0, oif = 0;
        struct fib6_info *from;
        struct dst_entry *dst;
        struct rt6_info *rt;
        struct sk_buff *skb;
        struct rtmsg *rtm;
        struct flowi6 fl6 = {};
        __be32 flowlabel;
        bool fibmatch;

        err = inet6_rtm_valid_getroute_req(in_skb, nlh, tb, extack);
        if (err < 0)
                goto errout;

        err = -EINVAL;
        rtm = nlmsg_data(nlh);
        fibmatch = !!(rtm->rtm_flags & RTM_F_FIB_MATCH);

        if (tb[RTA_SRC]) {
                if (nla_len(tb[RTA_SRC]) < sizeof(struct in6_addr))
                        goto errout;

                fl6.saddr = *(struct in6_addr *)nla_data(tb[RTA_SRC]);
        }

        if (tb[RTA_DST]) {
                if (nla_len(tb[RTA_DST]) < sizeof(struct in6_addr))
                        goto errout;

                fl6.daddr = *(struct in6_addr *)nla_data(tb[RTA_DST]);
        }

        if (tb[RTA_IIF])
                iif = nla_get_u32(tb[RTA_IIF]);

        if (tb[RTA_OIF])
                oif = nla_get_u32(tb[RTA_OIF]);

        if (tb[RTA_MARK])
                fl6.flowi6_mark = nla_get_u32(tb[RTA_MARK]);

        if (tb[RTA_UID])
                fl6.flowi6_uid = make_kuid(current_user_ns(),
                                           nla_get_u32(tb[RTA_UID]));
        else
                fl6.flowi6_uid = iif ? INVALID_UID : current_uid();

        if (tb[RTA_SPORT])
                fl6.fl6_sport = nla_get_be16(tb[RTA_SPORT]);

        if (tb[RTA_DPORT])
                fl6.fl6_dport = nla_get_be16(tb[RTA_DPORT]);

        if (tb[RTA_IP_PROTO]) {
                err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO],
                                                  &fl6.flowi6_proto, AF_INET6,
                                                  extack);
                if (err)
                        goto errout;
        }

        flowlabel = nla_get_be32_default(tb[RTA_FLOWLABEL], 0);
        fl6.flowlabel = ip6_make_flowinfo(rtm->rtm_tos, flowlabel);

        if (iif) {
                struct net_device *dev;
                int flags = 0;

                rcu_read_lock();

                dev = dev_get_by_index_rcu(net, iif);
                if (!dev) {
                        rcu_read_unlock();
                        err = -ENODEV;
                        goto errout;
                }

                fl6.flowi6_iif = iif;

                if (!ipv6_addr_any(&fl6.saddr))
                        flags |= RT6_LOOKUP_F_HAS_SADDR;

                dst = ip6_route_input_lookup(net, dev, &fl6, NULL, flags);

                rcu_read_unlock();
        } else {
                fl6.flowi6_oif = oif;

                dst = ip6_route_output(net, NULL, &fl6);
        }


        rt = dst_rt6_info(dst);
        if (rt->dst.error) {
                err = rt->dst.error;
                ip6_rt_put(rt);
                goto errout;
        }

        if (rt == net->ipv6.ip6_null_entry) {
                err = rt->dst.error;
                ip6_rt_put(rt);
                goto errout;
        }

        skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
        if (!skb) {
                ip6_rt_put(rt);
                err = -ENOBUFS;
                goto errout;
        }

        skb_dst_set(skb, &rt->dst);

        rcu_read_lock();
        from = rcu_dereference(rt->from);
        if (from) {
                if (fibmatch)
                        err = rt6_fill_node(net, skb, from, NULL, NULL, NULL,
                                            iif, RTM_NEWROUTE,
                                            NETLINK_CB(in_skb).portid,
                                            nlh->nlmsg_seq, 0);
                else
                        err = rt6_fill_node(net, skb, from, dst, &fl6.daddr,
                                            &fl6.saddr, iif, RTM_NEWROUTE,
                                            NETLINK_CB(in_skb).portid,
                                            nlh->nlmsg_seq, 0);
        } else {
                err = -ENETUNREACH;
        }
        rcu_read_unlock();

        if (err < 0) {
                kfree_skb(skb);
                goto errout;
        }

        err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
errout:
        return err;
}

void inet6_rt_notify(int event, struct fib6_info *rt, struct nl_info *info,
                     unsigned int nlm_flags)
{
        struct sk_buff *skb;
        struct net *net = info->nl_net;
        u32 seq;
        int err;

        err = -ENOBUFS;
        seq = info->nlh ? info->nlh->nlmsg_seq : 0;

        skb = nlmsg_new(rt6_nlmsg_size(rt), GFP_ATOMIC);
        if (!skb)
                goto errout;

        err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, 0,
                            event, info->portid, seq, nlm_flags);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }
        rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE,
                    info->nlh, GFP_ATOMIC);
        return;
errout:
        rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err);
}

void fib6_rt_update(struct net *net, struct fib6_info *rt,
                    struct nl_info *info)
{
        u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
        struct sk_buff *skb;
        int err = -ENOBUFS;

        skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any());
        if (!skb)
                goto errout;

        err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, 0,
                            RTM_NEWROUTE, info->portid, seq, NLM_F_REPLACE);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }
        rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE,
                    info->nlh, gfp_any());
        return;
errout:
        rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err);
}

void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i,
                            bool offload, bool trap, bool offload_failed)
{
        struct sk_buff *skb;
        int err;

        if (READ_ONCE(f6i->offload) == offload &&
            READ_ONCE(f6i->trap) == trap &&
            READ_ONCE(f6i->offload_failed) == offload_failed)
                return;

        WRITE_ONCE(f6i->offload, offload);
        WRITE_ONCE(f6i->trap, trap);

        /* 2 means send notifications only if offload_failed was changed. */
        if (net->ipv6.sysctl.fib_notify_on_flag_change == 2 &&
            READ_ONCE(f6i->offload_failed) == offload_failed)
                return;

        WRITE_ONCE(f6i->offload_failed, offload_failed);

        if (!rcu_access_pointer(f6i->fib6_node))
                /* The route was removed from the tree, do not send
                 * notification.
                 */
                return;

        if (!net->ipv6.sysctl.fib_notify_on_flag_change)
                return;

        skb = nlmsg_new(rt6_nlmsg_size(f6i), GFP_KERNEL);
        if (!skb) {
                err = -ENOBUFS;
                goto errout;
        }

        err = rt6_fill_node(net, skb, f6i, NULL, NULL, NULL, 0, RTM_NEWROUTE, 0,
                            0, 0);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }

        rtnl_notify(skb, net, 0, RTNLGRP_IPV6_ROUTE, NULL, GFP_KERNEL);
        return;

errout:
        rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err);
}
EXPORT_SYMBOL(fib6_info_hw_flags_set);

static int ip6_route_dev_notify(struct notifier_block *this,
                                unsigned long event, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct net *net = dev_net(dev);

        if (!(dev->flags & IFF_LOOPBACK))
                return NOTIFY_OK;

        if (event == NETDEV_REGISTER) {
                net->ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = dev;
                net->ipv6.ip6_null_entry->dst.dev = dev;
                net->ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(dev);
#ifdef CONFIG_IPV6_MULTIPLE_TABLES
                net->ipv6.ip6_prohibit_entry->dst.dev = dev;
                net->ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(dev);
                net->ipv6.ip6_blk_hole_entry->dst.dev = dev;
                net->ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(dev);
#endif
         } else if (event == NETDEV_UNREGISTER &&
                    dev->reg_state != NETREG_UNREGISTERED) {
                /* NETDEV_UNREGISTER could be fired for multiple times by
                 * netdev_wait_allrefs(). Make sure we only call this once.
                 */
                in6_dev_put_clear(&net->ipv6.ip6_null_entry->rt6i_idev);
#ifdef CONFIG_IPV6_MULTIPLE_TABLES
                in6_dev_put_clear(&net->ipv6.ip6_prohibit_entry->rt6i_idev);
                in6_dev_put_clear(&net->ipv6.ip6_blk_hole_entry->rt6i_idev);
#endif
        }

        return NOTIFY_OK;
}

/*
 *        /proc
 */

#ifdef CONFIG_PROC_FS
static int rt6_stats_seq_show(struct seq_file *seq, void *v)
{
        struct net *net = (struct net *)seq->private;
        seq_printf(seq, "%04x %04x %04x %04x %04x %04x %04x\n",
                   net->ipv6.rt6_stats->fib_nodes,
                   net->ipv6.rt6_stats->fib_route_nodes,
                   atomic_read(&net->ipv6.rt6_stats->fib_rt_alloc),
                   net->ipv6.rt6_stats->fib_rt_entries,
                   net->ipv6.rt6_stats->fib_rt_cache,
                   dst_entries_get_slow(&net->ipv6.ip6_dst_ops),
                   net->ipv6.rt6_stats->fib_discarded_routes);

        return 0;
}
#endif        /* CONFIG_PROC_FS */

#ifdef CONFIG_SYSCTL

static int ipv6_sysctl_rtcache_flush(const struct ctl_table *ctl, int write,
                              void *buffer, size_t *lenp, loff_t *ppos)
{
        struct net *net;
        int delay;
        int ret;
        if (!write)
                return -EINVAL;

        ret = proc_dointvec(ctl, write, buffer, lenp, ppos);
        if (ret)
                return ret;

        net = (struct net *)ctl->extra1;
        delay = net->ipv6.sysctl.flush_delay;
        fib6_run_gc(delay <= 0 ? 0 : (unsigned long)delay, net, delay > 0);
        return 0;
}

static struct ctl_table ipv6_route_table_template[] = {
        {
                .procname        =        "max_size",
                .data                =        &init_net.ipv6.sysctl.ip6_rt_max_size,
                .maxlen                =        sizeof(int),
                .mode                =        0644,
                .proc_handler        =        proc_dointvec,
        },
        {
                .procname        =        "gc_thresh",
                .data                =        &ip6_dst_ops_template.gc_thresh,
                .maxlen                =        sizeof(int),
                .mode                =        0644,
                .proc_handler        =        proc_dointvec,
        },
        {
                .procname        =        "flush",
                .data                =        &init_net.ipv6.sysctl.flush_delay,
                .maxlen                =        sizeof(int),
                .mode                =        0200,
                .proc_handler        =        ipv6_sysctl_rtcache_flush
        },
        {
                .procname        =        "gc_min_interval",
                .data                =        &init_net.ipv6.sysctl.ip6_rt_gc_min_interval,
                .maxlen                =        sizeof(int),
                .mode                =        0644,
                .proc_handler        =        proc_dointvec_jiffies,
        },
        {
                .procname        =        "gc_timeout",
                .data                =        &init_net.ipv6.sysctl.ip6_rt_gc_timeout,
                .maxlen                =        sizeof(int),
                .mode                =        0644,
                .proc_handler        =        proc_dointvec_jiffies,
        },
        {
                .procname        =        "gc_interval",
                .data                =        &init_net.ipv6.sysctl.ip6_rt_gc_interval,
                .maxlen                =        sizeof(int),
                .mode                =        0644,
                .proc_handler        =        proc_dointvec_jiffies,
        },
        {
                .procname        =        "gc_elasticity",
                .data                =        &init_net.ipv6.sysctl.ip6_rt_gc_elasticity,
                .maxlen                =        sizeof(int),
                .mode                =        0644,
                .proc_handler        =        proc_dointvec,
        },
        {
                .procname        =        "mtu_expires",
                .data                =        &init_net.ipv6.sysctl.ip6_rt_mtu_expires,
                .maxlen                =        sizeof(int),
                .mode                =        0644,
                .proc_handler        =        proc_dointvec_jiffies,
        },
        {
                .procname        =        "min_adv_mss",
                .data                =        &init_net.ipv6.sysctl.ip6_rt_min_advmss,
                .maxlen                =        sizeof(int),
                .mode                =        0644,
                .proc_handler        =        proc_dointvec,
        },
        {
                .procname        =        "gc_min_interval_ms",
                .data                =        &init_net.ipv6.sysctl.ip6_rt_gc_min_interval,
                .maxlen                =        sizeof(int),
                .mode                =        0644,
                .proc_handler        =        proc_dointvec_ms_jiffies,
        },
        {
                .procname        =        "skip_notify_on_dev_down",
                .data                =        &init_net.ipv6.sysctl.skip_notify_on_dev_down,
                .maxlen                =        sizeof(u8),
                .mode                =        0644,
                .proc_handler        =        proc_dou8vec_minmax,
                .extra1                =        SYSCTL_ZERO,
                .extra2                =        SYSCTL_ONE,
        },
};

struct ctl_table * __net_init ipv6_route_sysctl_init(struct net *net)
{
        struct ctl_table *table;

        table = kmemdup(ipv6_route_table_template,
                        sizeof(ipv6_route_table_template),
                        GFP_KERNEL);

        if (table) {
                table[0].data = &net->ipv6.sysctl.ip6_rt_max_size;
                table[1].data = &net->ipv6.ip6_dst_ops.gc_thresh;
                table[2].data = &net->ipv6.sysctl.flush_delay;
                table[2].extra1 = net;
                table[3].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval;
                table[4].data = &net->ipv6.sysctl.ip6_rt_gc_timeout;
                table[5].data = &net->ipv6.sysctl.ip6_rt_gc_interval;
                table[6].data = &net->ipv6.sysctl.ip6_rt_gc_elasticity;
                table[7].data = &net->ipv6.sysctl.ip6_rt_mtu_expires;
                table[8].data = &net->ipv6.sysctl.ip6_rt_min_advmss;
                table[9].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval;
                table[10].data = &net->ipv6.sysctl.skip_notify_on_dev_down;
        }

        return table;
}

size_t ipv6_route_sysctl_table_size(struct net *net)
{
        /* Don't export sysctls to unprivileged users */
        if (net->user_ns != &init_user_ns)
                return 1;

        return ARRAY_SIZE(ipv6_route_table_template);
}
#endif

static int __net_init ip6_route_net_init(struct net *net)
{
        int ret = -ENOMEM;

        memcpy(&net->ipv6.ip6_dst_ops, &ip6_dst_ops_template,
               sizeof(net->ipv6.ip6_dst_ops));

        if (dst_entries_init(&net->ipv6.ip6_dst_ops) < 0)
                goto out_ip6_dst_ops;

        net->ipv6.fib6_null_entry = fib6_info_alloc(GFP_KERNEL, true);
        if (!net->ipv6.fib6_null_entry)
                goto out_ip6_dst_entries;
        memcpy(net->ipv6.fib6_null_entry, &fib6_null_entry_template,
               sizeof(*net->ipv6.fib6_null_entry));

        net->ipv6.ip6_null_entry = kmemdup(&ip6_null_entry_template,
                                           sizeof(*net->ipv6.ip6_null_entry),
                                           GFP_KERNEL);
        if (!net->ipv6.ip6_null_entry)
                goto out_fib6_null_entry;
        net->ipv6.ip6_null_entry->dst.ops = &net->ipv6.ip6_dst_ops;
        dst_init_metrics(&net->ipv6.ip6_null_entry->dst,
                         ip6_template_metrics, true);
        INIT_LIST_HEAD(&net->ipv6.ip6_null_entry->dst.rt_uncached);

#ifdef CONFIG_IPV6_MULTIPLE_TABLES
        net->ipv6.fib6_has_custom_rules = false;
        net->ipv6.ip6_prohibit_entry = kmemdup(&ip6_prohibit_entry_template,
                                               sizeof(*net->ipv6.ip6_prohibit_entry),
                                               GFP_KERNEL);
        if (!net->ipv6.ip6_prohibit_entry)
                goto out_ip6_null_entry;
        net->ipv6.ip6_prohibit_entry->dst.ops = &net->ipv6.ip6_dst_ops;
        dst_init_metrics(&net->ipv6.ip6_prohibit_entry->dst,
                         ip6_template_metrics, true);
        INIT_LIST_HEAD(&net->ipv6.ip6_prohibit_entry->dst.rt_uncached);

        net->ipv6.ip6_blk_hole_entry = kmemdup(&ip6_blk_hole_entry_template,
                                               sizeof(*net->ipv6.ip6_blk_hole_entry),
                                               GFP_KERNEL);
        if (!net->ipv6.ip6_blk_hole_entry)
                goto out_ip6_prohibit_entry;
        net->ipv6.ip6_blk_hole_entry->dst.ops = &net->ipv6.ip6_dst_ops;
        dst_init_metrics(&net->ipv6.ip6_blk_hole_entry->dst,
                         ip6_template_metrics, true);
        INIT_LIST_HEAD(&net->ipv6.ip6_blk_hole_entry->dst.rt_uncached);
#ifdef CONFIG_IPV6_SUBTREES
        net->ipv6.fib6_routes_require_src = 0;
#endif
#endif

        net->ipv6.sysctl.flush_delay = 0;
        net->ipv6.sysctl.ip6_rt_max_size = INT_MAX;
        net->ipv6.sysctl.ip6_rt_gc_min_interval = HZ / 2;
        net->ipv6.sysctl.ip6_rt_gc_timeout = 60*HZ;
        net->ipv6.sysctl.ip6_rt_gc_interval = 30*HZ;
        net->ipv6.sysctl.ip6_rt_gc_elasticity = 9;
        net->ipv6.sysctl.ip6_rt_mtu_expires = 10*60*HZ;
        net->ipv6.sysctl.ip6_rt_min_advmss = IPV6_MIN_MTU - 20 - 40;
        net->ipv6.sysctl.skip_notify_on_dev_down = 0;

        atomic_set(&net->ipv6.ip6_rt_gc_expire, 30*HZ);

        ret = 0;
out:
        return ret;

#ifdef CONFIG_IPV6_MULTIPLE_TABLES
out_ip6_prohibit_entry:
        kfree(net->ipv6.ip6_prohibit_entry);
out_ip6_null_entry:
        kfree(net->ipv6.ip6_null_entry);
#endif
out_fib6_null_entry:
        kfree(net->ipv6.fib6_null_entry);
out_ip6_dst_entries:
        dst_entries_destroy(&net->ipv6.ip6_dst_ops);
out_ip6_dst_ops:
        goto out;
}

static void __net_exit ip6_route_net_exit(struct net *net)
{
        kfree(net->ipv6.fib6_null_entry);
        kfree(net->ipv6.ip6_null_entry);
#ifdef CONFIG_IPV6_MULTIPLE_TABLES
        kfree(net->ipv6.ip6_prohibit_entry);
        kfree(net->ipv6.ip6_blk_hole_entry);
#endif
        dst_entries_destroy(&net->ipv6.ip6_dst_ops);
}

static int __net_init ip6_route_net_init_late(struct net *net)
{
#ifdef CONFIG_PROC_FS
        if (!proc_create_net("ipv6_route", 0, net->proc_net,
                             &ipv6_route_seq_ops,
                             sizeof(struct ipv6_route_iter)))
                return -ENOMEM;

        if (!proc_create_net_single("rt6_stats", 0444, net->proc_net,
                                    rt6_stats_seq_show, NULL)) {
                remove_proc_entry("ipv6_route", net->proc_net);
                return -ENOMEM;
        }
#endif
        return 0;
}

static void __net_exit ip6_route_net_exit_late(struct net *net)
{
#ifdef CONFIG_PROC_FS
        remove_proc_entry("ipv6_route", net->proc_net);
        remove_proc_entry("rt6_stats", net->proc_net);
#endif
}

static struct pernet_operations ip6_route_net_ops = {
        .init = ip6_route_net_init,
        .exit = ip6_route_net_exit,
};

static int __net_init ipv6_inetpeer_init(struct net *net)
{
        struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);

        if (!bp)
                return -ENOMEM;
        inet_peer_base_init(bp);
        net->ipv6.peers = bp;
        return 0;
}

static void __net_exit ipv6_inetpeer_exit(struct net *net)
{
        struct inet_peer_base *bp = net->ipv6.peers;

        net->ipv6.peers = NULL;
        inetpeer_invalidate_tree(bp);
        kfree(bp);
}

static struct pernet_operations ipv6_inetpeer_ops = {
        .init        =        ipv6_inetpeer_init,
        .exit        =        ipv6_inetpeer_exit,
};

static struct pernet_operations ip6_route_net_late_ops = {
        .init = ip6_route_net_init_late,
        .exit = ip6_route_net_exit_late,
};

static struct notifier_block ip6_route_dev_notifier = {
        .notifier_call = ip6_route_dev_notify,
        .priority = ADDRCONF_NOTIFY_PRIORITY - 10,
};

void __init ip6_route_init_special_entries(void)
{
        /* Registering of the loopback is done before this portion of code,
         * the loopback reference in rt6_info will not be taken, do it
         * manually for init_net */
        init_net.ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = init_net.loopback_dev;
        init_net.ipv6.ip6_null_entry->dst.dev = init_net.loopback_dev;
        init_net.ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev);
  #ifdef CONFIG_IPV6_MULTIPLE_TABLES
        init_net.ipv6.ip6_prohibit_entry->dst.dev = init_net.loopback_dev;
        init_net.ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev);
        init_net.ipv6.ip6_blk_hole_entry->dst.dev = init_net.loopback_dev;
        init_net.ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev);
  #endif
}

#if IS_BUILTIN(CONFIG_IPV6)
#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
DEFINE_BPF_ITER_FUNC(ipv6_route, struct bpf_iter_meta *meta, struct fib6_info *rt)

BTF_ID_LIST(btf_fib6_info_id)
BTF_ID(struct, fib6_info)

static const struct bpf_iter_seq_info ipv6_route_seq_info = {
        .seq_ops                = &ipv6_route_seq_ops,
        .init_seq_private        = bpf_iter_init_seq_net,
        .fini_seq_private        = bpf_iter_fini_seq_net,
        .seq_priv_size                = sizeof(struct ipv6_route_iter),
};

static struct bpf_iter_reg ipv6_route_reg_info = {
        .target                        = "ipv6_route",
        .ctx_arg_info_size        = 1,
        .ctx_arg_info                = {
                { offsetof(struct bpf_iter__ipv6_route, rt),
                  PTR_TO_BTF_ID_OR_NULL },
        },
        .seq_info                = &ipv6_route_seq_info,
};

static int __init bpf_iter_register(void)
{
        ipv6_route_reg_info.ctx_arg_info[0].btf_id = *btf_fib6_info_id;
        return bpf_iter_reg_target(&ipv6_route_reg_info);
}

static void bpf_iter_unregister(void)
{
        bpf_iter_unreg_target(&ipv6_route_reg_info);
}
#endif
#endif

static const struct rtnl_msg_handler ip6_route_rtnl_msg_handlers[] __initconst_or_module = {
        {.owner = THIS_MODULE, .protocol = PF_INET6, .msgtype = RTM_NEWROUTE,
         .doit = inet6_rtm_newroute},
        {.owner = THIS_MODULE, .protocol = PF_INET6, .msgtype = RTM_DELROUTE,
         .doit = inet6_rtm_delroute},
        {.owner = THIS_MODULE, .protocol = PF_INET6, .msgtype = RTM_GETROUTE,
         .doit = inet6_rtm_getroute, .flags = RTNL_FLAG_DOIT_UNLOCKED},
};

int __init ip6_route_init(void)
{
        int ret;
        int cpu;

        ret = -ENOMEM;
        ip6_dst_ops_template.kmem_cachep =
                kmem_cache_create("ip6_dst_cache", sizeof(struct rt6_info), 0,
                                  SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL);
        if (!ip6_dst_ops_template.kmem_cachep)
                goto out;

        ret = dst_entries_init(&ip6_dst_blackhole_ops);
        if (ret)
                goto out_kmem_cache;

        ret = register_pernet_subsys(&ipv6_inetpeer_ops);
        if (ret)
                goto out_dst_entries;

        ret = register_pernet_subsys(&ip6_route_net_ops);
        if (ret)
                goto out_register_inetpeer;

        ip6_dst_blackhole_ops.kmem_cachep = ip6_dst_ops_template.kmem_cachep;

        ret = fib6_init();
        if (ret)
                goto out_register_subsys;

        ret = xfrm6_init();
        if (ret)
                goto out_fib6_init;

        ret = fib6_rules_init();
        if (ret)
                goto xfrm6_init;

        ret = register_pernet_subsys(&ip6_route_net_late_ops);
        if (ret)
                goto fib6_rules_init;

        ret = rtnl_register_many(ip6_route_rtnl_msg_handlers);
        if (ret < 0)
                goto out_register_late_subsys;

        ret = register_netdevice_notifier(&ip6_route_dev_notifier);
        if (ret)
                goto out_register_late_subsys;

#if IS_BUILTIN(CONFIG_IPV6)
#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
        ret = bpf_iter_register();
        if (ret)
                goto out_register_late_subsys;
#endif
#endif

        for_each_possible_cpu(cpu) {
                struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu);

                INIT_LIST_HEAD(&ul->head);
                spin_lock_init(&ul->lock);
        }

out:
        return ret;

out_register_late_subsys:
        rtnl_unregister_all(PF_INET6);
        unregister_pernet_subsys(&ip6_route_net_late_ops);
fib6_rules_init:
        fib6_rules_cleanup();
xfrm6_init:
        xfrm6_fini();
out_fib6_init:
        fib6_gc_cleanup();
out_register_subsys:
        unregister_pernet_subsys(&ip6_route_net_ops);
out_register_inetpeer:
        unregister_pernet_subsys(&ipv6_inetpeer_ops);
out_dst_entries:
        dst_entries_destroy(&ip6_dst_blackhole_ops);
out_kmem_cache:
        kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep);
        goto out;
}

void ip6_route_cleanup(void)
{
#if IS_BUILTIN(CONFIG_IPV6)
#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
        bpf_iter_unregister();
#endif
#endif
        unregister_netdevice_notifier(&ip6_route_dev_notifier);
        unregister_pernet_subsys(&ip6_route_net_late_ops);
        fib6_rules_cleanup();
        xfrm6_fini();
        fib6_gc_cleanup();
        unregister_pernet_subsys(&ipv6_inetpeer_ops);
        unregister_pernet_subsys(&ip6_route_net_ops);
        dst_entries_destroy(&ip6_dst_blackhole_ops);
        kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep);
}








































































   16 
   16 

   16 







    7 











    7 
    7 







































































    7 



    7 




    7 
    7 
    7 


    7 





















    7 



















   23 


   23 












1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
 */

#include "peer.h"
#include "device.h"
#include "queueing.h"
#include "timers.h"
#include "peerlookup.h"
#include "noise.h"

#include <linux/kref.h>
#include <linux/lockdep.h>
#include <linux/rcupdate.h>
#include <linux/list.h>

static struct kmem_cache *peer_cache;
static atomic64_t peer_counter = ATOMIC64_INIT(0);

struct wg_peer *wg_peer_create(struct wg_device *wg,
                               const u8 public_key[NOISE_PUBLIC_KEY_LEN],
                               const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN])
{
        struct wg_peer *peer;
        int ret = -ENOMEM;

        lockdep_assert_held(&wg->device_update_lock);

        if (wg->num_peers >= MAX_PEERS_PER_DEVICE)
                return ERR_PTR(ret);

        peer = kmem_cache_zalloc(peer_cache, GFP_KERNEL);
        if (unlikely(!peer))
                return ERR_PTR(ret);
        if (unlikely(dst_cache_init(&peer->endpoint_cache, GFP_KERNEL)))
                goto err;

        peer->device = wg;
        wg_noise_handshake_init(&peer->handshake, &wg->static_identity,
                                public_key, preshared_key, peer);
        peer->internal_id = atomic64_inc_return(&peer_counter);
        peer->serial_work_cpu = nr_cpumask_bits;
        wg_cookie_init(&peer->latest_cookie);
        wg_timers_init(peer);
        wg_cookie_checker_precompute_peer_keys(peer);
        spin_lock_init(&peer->keypairs.keypair_update_lock);
        INIT_WORK(&peer->transmit_handshake_work, wg_packet_handshake_send_worker);
        INIT_WORK(&peer->transmit_packet_work, wg_packet_tx_worker);
        wg_prev_queue_init(&peer->tx_queue);
        wg_prev_queue_init(&peer->rx_queue);
        rwlock_init(&peer->endpoint_lock);
        kref_init(&peer->refcount);
        skb_queue_head_init(&peer->staged_packet_queue);
        wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
        set_bit(NAPI_STATE_NO_BUSY_POLL, &peer->napi.state);
        netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll);
        napi_enable(&peer->napi);
        list_add_tail(&peer->peer_list, &wg->peer_list);
        INIT_LIST_HEAD(&peer->allowedips_list);
        wg_pubkey_hashtable_add(wg->peer_hashtable, peer);
        ++wg->num_peers;
        pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id);
        return peer;

err:
        kmem_cache_free(peer_cache, peer);
        return ERR_PTR(ret);
}

struct wg_peer *wg_peer_get_maybe_zero(struct wg_peer *peer)
{
        RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(),
                         "Taking peer reference without holding the RCU read lock");
        if (unlikely(!peer || !kref_get_unless_zero(&peer->refcount)))
                return NULL;
        return peer;
}

static void peer_make_dead(struct wg_peer *peer)
{
        /* Remove from configuration-time lookup structures. */
        list_del_init(&peer->peer_list);
        wg_allowedips_remove_by_peer(&peer->device->peer_allowedips, peer,
                                     &peer->device->device_update_lock);
        wg_pubkey_hashtable_remove(peer->device->peer_hashtable, peer);

        /* Mark as dead, so that we don't allow jumping contexts after. */
        WRITE_ONCE(peer->is_dead, true);

        /* The caller must now synchronize_net() for this to take effect. */
}

static void peer_remove_after_dead(struct wg_peer *peer)
{
        WARN_ON(!peer->is_dead);

        /* No more keypairs can be created for this peer, since is_dead protects
         * add_new_keypair, so we can now destroy existing ones.
         */
        wg_noise_keypairs_clear(&peer->keypairs);

        /* Destroy all ongoing timers that were in-flight at the beginning of
         * this function.
         */
        wg_timers_stop(peer);

        /* The transition between packet encryption/decryption queues isn't
         * guarded by is_dead, but each reference's life is strictly bounded by
         * two generations: once for parallel crypto and once for serial
         * ingestion, so we can simply flush twice, and be sure that we no
         * longer have references inside these queues.
         */

        /* a) For encrypt/decrypt. */
        flush_workqueue(peer->device->packet_crypt_wq);
        /* b.1) For send (but not receive, since that's napi). */
        flush_workqueue(peer->device->packet_crypt_wq);
        /* b.2.1) For receive (but not send, since that's wq). */
        napi_disable(&peer->napi);
        /* b.2.1) It's now safe to remove the napi struct, which must be done
         * here from process context.
         */
        netif_napi_del(&peer->napi);

        /* Ensure any workstructs we own (like transmit_handshake_work or
         * clear_peer_work) no longer are in use.
         */
        flush_workqueue(peer->device->handshake_send_wq);

        /* After the above flushes, a peer might still be active in a few
         * different contexts: 1) from xmit(), before hitting is_dead and
         * returning, 2) from wg_packet_consume_data(), before hitting is_dead
         * and returning, 3) from wg_receive_handshake_packet() after a point
         * where it has processed an incoming handshake packet, but where
         * all calls to pass it off to timers fails because of is_dead. We won't
         * have new references in (1) eventually, because we're removed from
         * allowedips; we won't have new references in (2) eventually, because
         * wg_index_hashtable_lookup will always return NULL, since we removed
         * all existing keypairs and no more can be created; we won't have new
         * references in (3) eventually, because we're removed from the pubkey
         * hash table, which allows for a maximum of one handshake response,
         * via the still-uncleared index hashtable entry, but not more than one,
         * and in wg_cookie_message_consume, the lookup eventually gets a peer
         * with a refcount of zero, so no new reference is taken.
         */

        --peer->device->num_peers;
        wg_peer_put(peer);
}

/* We have a separate "remove" function make sure that all active places where
 * a peer is currently operating will eventually come to an end and not pass
 * their reference onto another context.
 */
void wg_peer_remove(struct wg_peer *peer)
{
        if (unlikely(!peer))
                return;
        lockdep_assert_held(&peer->device->device_update_lock);

        peer_make_dead(peer);
        synchronize_net();
        peer_remove_after_dead(peer);
}

void wg_peer_remove_all(struct wg_device *wg)
{
        struct wg_peer *peer, *temp;
        LIST_HEAD(dead_peers);

        lockdep_assert_held(&wg->device_update_lock);

        /* Avoid having to traverse individually for each one. */
        wg_allowedips_free(&wg->peer_allowedips, &wg->device_update_lock);

        list_for_each_entry_safe(peer, temp, &wg->peer_list, peer_list) {
                peer_make_dead(peer);
                list_add_tail(&peer->peer_list, &dead_peers);
        }
        synchronize_net();
        list_for_each_entry_safe(peer, temp, &dead_peers, peer_list)
                peer_remove_after_dead(peer);
}

static void rcu_release(struct rcu_head *rcu)
{
        struct wg_peer *peer = container_of(rcu, struct wg_peer, rcu);

        dst_cache_destroy(&peer->endpoint_cache);
        WARN_ON(wg_prev_queue_peek(&peer->tx_queue) || wg_prev_queue_peek(&peer->rx_queue));

        /* The final zeroing takes care of clearing any remaining handshake key
         * material and other potentially sensitive information.
         */
        memzero_explicit(peer, sizeof(*peer));
        kmem_cache_free(peer_cache, peer);
}

static void kref_release(struct kref *refcount)
{
        struct wg_peer *peer = container_of(refcount, struct wg_peer, refcount);

        pr_debug("%s: Peer %llu (%pISpfsc) destroyed\n",
                 peer->device->dev->name, peer->internal_id,
                 &peer->endpoint.addr);

        /* Remove ourself from dynamic runtime lookup structures, now that the
         * last reference is gone.
         */
        wg_index_hashtable_remove(peer->device->index_hashtable,
                                  &peer->handshake.entry);

        /* Remove any lingering packets that didn't have a chance to be
         * transmitted.
         */
        wg_packet_purge_staged_packets(peer);

        /* Free the memory used. */
        call_rcu(&peer->rcu, rcu_release);
}

void wg_peer_put(struct wg_peer *peer)
{
        if (unlikely(!peer))
                return;
        kref_put(&peer->refcount, kref_release);
}

int __init wg_peer_init(void)
{
        peer_cache = KMEM_CACHE(wg_peer, 0);
        return peer_cache ? 0 : -ENOMEM;
}

void wg_peer_uninit(void)
{
        kmem_cache_destroy(peer_cache);
}



























































































































































































































  528 







 4957 

















 4081 
  294 

  127 



































































 1625 
















   74 
    3 










































 3425 












































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 *        Berkeley style UIO structures        -        Alan Cox 1994.
 */
#ifndef __LINUX_UIO_H
#define __LINUX_UIO_H

#include <linux/kernel.h>
#include <linux/mm_types.h>
#include <linux/ucopysize.h>
#include <uapi/linux/uio.h>

struct page;
struct folio_queue;

typedef unsigned int __bitwise iov_iter_extraction_t;

struct kvec {
        void *iov_base; /* and that should *never* hold a userland pointer */
        size_t iov_len;
};

enum iter_type {
        /* iter types */
        ITER_UBUF,
        ITER_IOVEC,
        ITER_BVEC,
        ITER_KVEC,
        ITER_FOLIOQ,
        ITER_XARRAY,
        ITER_DISCARD,
};

#define ITER_SOURCE        1        // == WRITE
#define ITER_DEST        0        // == READ

struct iov_iter_state {
        size_t iov_offset;
        size_t count;
        unsigned long nr_segs;
};

struct iov_iter {
        u8 iter_type;
        bool nofault;
        bool data_source;
        size_t iov_offset;
        /*
         * Hack alert: overlay ubuf_iovec with iovec + count, so
         * that the members resolve correctly regardless of the type
         * of iterator used. This means that you can use:
         *
         * &iter->__ubuf_iovec or iter->__iov
         *
         * interchangably for the user_backed cases, hence simplifying
         * some of the cases that need to deal with both.
         */
        union {
                /*
                 * This really should be a const, but we cannot do that without
                 * also modifying any of the zero-filling iter init functions.
                 * Leave it non-const for now, but it should be treated as such.
                 */
                struct iovec __ubuf_iovec;
                struct {
                        union {
                                /* use iter_iov() to get the current vec */
                                const struct iovec *__iov;
                                const struct kvec *kvec;
                                const struct bio_vec *bvec;
                                const struct folio_queue *folioq;
                                struct xarray *xarray;
                                void __user *ubuf;
                        };
                        size_t count;
                };
        };
        union {
                unsigned long nr_segs;
                u8 folioq_slot;
                loff_t xarray_start;
        };
};

typedef __u16 uio_meta_flags_t;

struct uio_meta {
        uio_meta_flags_t        flags;
        u16                        app_tag;
        u64                        seed;
        struct iov_iter                iter;
};

static inline const struct iovec *iter_iov(const struct iov_iter *iter)
{
        if (iter->iter_type == ITER_UBUF)
                return (const struct iovec *) &iter->__ubuf_iovec;
        return iter->__iov;
}

#define iter_iov_addr(iter)        (iter_iov(iter)->iov_base + (iter)->iov_offset)
#define iter_iov_len(iter)        (iter_iov(iter)->iov_len - (iter)->iov_offset)

static inline enum iter_type iov_iter_type(const struct iov_iter *i)
{
        return i->iter_type;
}

static inline void iov_iter_save_state(struct iov_iter *iter,
                                       struct iov_iter_state *state)
{
        state->iov_offset = iter->iov_offset;
        state->count = iter->count;
        state->nr_segs = iter->nr_segs;
}

static inline bool iter_is_ubuf(const struct iov_iter *i)
{
        return iov_iter_type(i) == ITER_UBUF;
}

static inline bool iter_is_iovec(const struct iov_iter *i)
{
        return iov_iter_type(i) == ITER_IOVEC;
}

static inline bool iov_iter_is_kvec(const struct iov_iter *i)
{
        return iov_iter_type(i) == ITER_KVEC;
}

static inline bool iov_iter_is_bvec(const struct iov_iter *i)
{
        return iov_iter_type(i) == ITER_BVEC;
}

static inline bool iov_iter_is_discard(const struct iov_iter *i)
{
        return iov_iter_type(i) == ITER_DISCARD;
}

static inline bool iov_iter_is_folioq(const struct iov_iter *i)
{
        return iov_iter_type(i) == ITER_FOLIOQ;
}

static inline bool iov_iter_is_xarray(const struct iov_iter *i)
{
        return iov_iter_type(i) == ITER_XARRAY;
}

static inline unsigned char iov_iter_rw(const struct iov_iter *i)
{
        return i->data_source ? WRITE : READ;
}

static inline bool user_backed_iter(const struct iov_iter *i)
{
        return iter_is_ubuf(i) || iter_is_iovec(i);
}

/*
 * Total number of bytes covered by an iovec.
 *
 * NOTE that it is not safe to use this function until all the iovec's
 * segment lengths have been validated.  Because the individual lengths can
 * overflow a size_t when added together.
 */
static inline size_t iov_length(const struct iovec *iov, unsigned long nr_segs)
{
        unsigned long seg;
        size_t ret = 0;

        for (seg = 0; seg < nr_segs; seg++)
                ret += iov[seg].iov_len;
        return ret;
}

size_t copy_page_from_iter_atomic(struct page *page, size_t offset,
                                  size_t bytes, struct iov_iter *i);
void iov_iter_advance(struct iov_iter *i, size_t bytes);
void iov_iter_revert(struct iov_iter *i, size_t bytes);
size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t bytes);
size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t bytes);
size_t iov_iter_single_seg_count(const struct iov_iter *i);
size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
                         struct iov_iter *i);
size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
                         struct iov_iter *i);

size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i);
size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i);
size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i);

static inline size_t copy_folio_to_iter(struct folio *folio, size_t offset,
                size_t bytes, struct iov_iter *i)
{
        return copy_page_to_iter(&folio->page, offset, bytes, i);
}

static inline size_t copy_folio_from_iter(struct folio *folio, size_t offset,
                                          size_t bytes, struct iov_iter *i)
{
        return copy_page_from_iter(&folio->page, offset, bytes, i);
}

static inline size_t copy_folio_from_iter_atomic(struct folio *folio,
                size_t offset, size_t bytes, struct iov_iter *i)
{
        return copy_page_from_iter_atomic(&folio->page, offset, bytes, i);
}

size_t copy_page_to_iter_nofault(struct page *page, unsigned offset,
                                 size_t bytes, struct iov_iter *i);

static __always_inline __must_check
size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
{
        if (check_copy_size(addr, bytes, true))
                return _copy_to_iter(addr, bytes, i);
        return 0;
}

static __always_inline __must_check
size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
{
        if (check_copy_size(addr, bytes, false))
                return _copy_from_iter(addr, bytes, i);
        return 0;
}

static __always_inline __must_check
bool copy_to_iter_full(const void *addr, size_t bytes, struct iov_iter *i)
{
        size_t copied = copy_to_iter(addr, bytes, i);
        if (likely(copied == bytes))
                return true;
        iov_iter_revert(i, copied);
        return false;
}

static __always_inline __must_check
bool copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
{
        size_t copied = copy_from_iter(addr, bytes, i);
        if (likely(copied == bytes))
                return true;
        iov_iter_revert(i, copied);
        return false;
}

static __always_inline __must_check
size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
{
        if (check_copy_size(addr, bytes, false))
                return _copy_from_iter_nocache(addr, bytes, i);
        return 0;
}

static __always_inline __must_check
bool copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
{
        size_t copied = copy_from_iter_nocache(addr, bytes, i);
        if (likely(copied == bytes))
                return true;
        iov_iter_revert(i, copied);
        return false;
}

#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
/*
 * Note, users like pmem that depend on the stricter semantics of
 * _copy_from_iter_flushcache() than _copy_from_iter_nocache() must check for
 * IS_ENABLED(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) before assuming that the
 * destination is flushed from the cache on return.
 */
size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i);
#else
#define _copy_from_iter_flushcache _copy_from_iter_nocache
#endif

#ifdef CONFIG_ARCH_HAS_COPY_MC
size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i);
#else
#define _copy_mc_to_iter _copy_to_iter
#endif

size_t iov_iter_zero(size_t bytes, struct iov_iter *);
bool iov_iter_is_aligned(const struct iov_iter *i, unsigned addr_mask,
                        unsigned len_mask);
unsigned long iov_iter_alignment(const struct iov_iter *i);
unsigned long iov_iter_gap_alignment(const struct iov_iter *i);
void iov_iter_init(struct iov_iter *i, unsigned int direction, const struct iovec *iov,
                        unsigned long nr_segs, size_t count);
void iov_iter_kvec(struct iov_iter *i, unsigned int direction, const struct kvec *kvec,
                        unsigned long nr_segs, size_t count);
void iov_iter_bvec(struct iov_iter *i, unsigned int direction, const struct bio_vec *bvec,
                        unsigned long nr_segs, size_t count);
void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count);
void iov_iter_folio_queue(struct iov_iter *i, unsigned int direction,
                          const struct folio_queue *folioq,
                          unsigned int first_slot, unsigned int offset, size_t count);
void iov_iter_xarray(struct iov_iter *i, unsigned int direction, struct xarray *xarray,
                     loff_t start, size_t count);
ssize_t iov_iter_get_pages2(struct iov_iter *i, struct page **pages,
                        size_t maxsize, unsigned maxpages, size_t *start);
ssize_t iov_iter_get_pages_alloc2(struct iov_iter *i, struct page ***pages,
                        size_t maxsize, size_t *start);
int iov_iter_npages(const struct iov_iter *i, int maxpages);
void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state);

const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags);

static inline size_t iov_iter_count(const struct iov_iter *i)
{
        return i->count;
}

/*
 * Cap the iov_iter by given limit; note that the second argument is
 * *not* the new size - it's upper limit for such.  Passing it a value
 * greater than the amount of data in iov_iter is fine - it'll just do
 * nothing in that case.
 */
static inline void iov_iter_truncate(struct iov_iter *i, u64 count)
{
        /*
         * count doesn't have to fit in size_t - comparison extends both
         * operands to u64 here and any value that would be truncated by
         * conversion in assignement is by definition greater than all
         * values of size_t, including old i->count.
         */
        if (i->count > count)
                i->count = count;
}

/*
 * reexpand a previously truncated iterator; count must be no more than how much
 * we had shrunk it.
 */
static inline void iov_iter_reexpand(struct iov_iter *i, size_t count)
{
        i->count = count;
}

static inline int
iov_iter_npages_cap(struct iov_iter *i, int maxpages, size_t max_bytes)
{
        size_t shorted = 0;
        int npages;

        if (iov_iter_count(i) > max_bytes) {
                shorted = iov_iter_count(i) - max_bytes;
                iov_iter_truncate(i, max_bytes);
        }
        npages = iov_iter_npages(i, maxpages);
        if (shorted)
                iov_iter_reexpand(i, iov_iter_count(i) + shorted);

        return npages;
}

struct iovec *iovec_from_user(const struct iovec __user *uvector,
                unsigned long nr_segs, unsigned long fast_segs,
                struct iovec *fast_iov, bool compat);
ssize_t import_iovec(int type, const struct iovec __user *uvec,
                 unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
                 struct iov_iter *i);
ssize_t __import_iovec(int type, const struct iovec __user *uvec,
                 unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
                 struct iov_iter *i, bool compat);
int import_ubuf(int type, void __user *buf, size_t len, struct iov_iter *i);

static inline void iov_iter_ubuf(struct iov_iter *i, unsigned int direction,
                        void __user *buf, size_t count)
{
        WARN_ON(direction & ~(READ | WRITE));
        *i = (struct iov_iter) {
                .iter_type = ITER_UBUF,
                .data_source = direction,
                .ubuf = buf,
                .count = count,
                .nr_segs = 1
        };
}
/* Flags for iov_iter_get/extract_pages*() */
/* Allow P2PDMA on the extracted pages */
#define ITER_ALLOW_P2PDMA        ((__force iov_iter_extraction_t)0x01)

ssize_t iov_iter_extract_pages(struct iov_iter *i, struct page ***pages,
                               size_t maxsize, unsigned int maxpages,
                               iov_iter_extraction_t extraction_flags,
                               size_t *offset0);

/**
 * iov_iter_extract_will_pin - Indicate how pages from the iterator will be retained
 * @iter: The iterator
 *
 * Examine the iterator and indicate by returning true or false as to how, if
 * at all, pages extracted from the iterator will be retained by the extraction
 * function.
 *
 * %true indicates that the pages will have a pin placed in them that the
 * caller must unpin.  This is must be done for DMA/async DIO to force fork()
 * to forcibly copy a page for the child (the parent must retain the original
 * page).
 *
 * %false indicates that no measures are taken and that it's up to the caller
 * to retain the pages.
 */
static inline bool iov_iter_extract_will_pin(const struct iov_iter *iter)
{
        return user_backed_iter(iter);
}

struct sg_table;
ssize_t extract_iter_to_sg(struct iov_iter *iter, size_t len,
                           struct sg_table *sgtable, unsigned int sg_max,
                           iov_iter_extraction_t extraction_flags);

#endif












































































































   63 

   75 







   36 

    3 





















   50 














   35 

   35 






   82 























   83 







   76 





   76 

   46 






   57 






   57 

   42 
   42 















   79 


   56 

   96 












   18 
    3 










   16 

   22 













   53 




   30 

   29 












   53 














   53 












   49 










   35 
    4 

   36 
   36 
   36 


   36 






   37 





   43 







   47 








   23 


   75 


   76 
   34 


   76 



   34 





   35 

    2 
   35 






    6 
   33 





   45 




   20 






   14 







    3 





   35 






   37 







   35 



    6 


   28 












   85 
   36 
















  114 
   98 

















   56 
    2 







   93 

   73 







    1 



   44 

    6 











   75 









   93 

   52 

   72 

   85 
   34 


   36 
   36 


   73 


    3 








   19 






   10 










   17 







   17 

















    2 

    2 



































   75 




















   75 











   97 










   97 







   50 



   42 

   69 


   34 










   98 





   12 


   96 


   96 



   49 


   45 






   45 




   49 
   49 





    4 


   36 
   36 
   36 








   70 


    4 


   18 

   34 
























































   79 









    7 




   77 


   17 









   10 










    6 
    6 





    6 


    1 

    1 
    1 


    1 










    3 













    1 



    2 
    1 


    3 
    1 









1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
/*
 * net/tipc/group.c: TIPC group messaging code
 *
 * Copyright (c) 2017, Ericsson AB
 * Copyright (c) 2020, Red Hat Inc
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the names of the copyright holders nor the names of its
 *    contributors may be used to endorse or promote products derived from
 *    this software without specific prior written permission.
 *
 * Alternatively, this software may be distributed under the terms of the
 * GNU General Public License ("GPL") version 2 as published by the Free
 * Software Foundation.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include "core.h"
#include "addr.h"
#include "group.h"
#include "bcast.h"
#include "topsrv.h"
#include "msg.h"
#include "socket.h"
#include "node.h"
#include "name_table.h"
#include "subscr.h"

#define ADV_UNIT (((MAX_MSG_SIZE + MAX_H_SIZE) / FLOWCTL_BLK_SZ) + 1)
#define ADV_IDLE ADV_UNIT
#define ADV_ACTIVE (ADV_UNIT * 12)

enum mbr_state {
        MBR_JOINING,
        MBR_PUBLISHED,
        MBR_JOINED,
        MBR_PENDING,
        MBR_ACTIVE,
        MBR_RECLAIMING,
        MBR_REMITTED,
        MBR_LEAVING
};

struct tipc_member {
        struct rb_node tree_node;
        struct list_head list;
        struct list_head small_win;
        struct sk_buff_head deferredq;
        struct tipc_group *group;
        u32 node;
        u32 port;
        u32 instance;
        enum mbr_state state;
        u16 advertised;
        u16 window;
        u16 bc_rcv_nxt;
        u16 bc_syncpt;
        u16 bc_acked;
};

struct tipc_group {
        struct rb_root members;
        struct list_head small_win;
        struct list_head pending;
        struct list_head active;
        struct tipc_nlist dests;
        struct net *net;
        int subid;
        u32 type;
        u32 instance;
        u32 scope;
        u32 portid;
        u16 member_cnt;
        u16 active_cnt;
        u16 max_active;
        u16 bc_snd_nxt;
        u16 bc_ackers;
        bool *open;
        bool loopback;
        bool events;
};

static void tipc_group_proto_xmit(struct tipc_group *grp, struct tipc_member *m,
                                  int mtyp, struct sk_buff_head *xmitq);

static void tipc_group_open(struct tipc_member *m, bool *wakeup)
{
        *wakeup = false;
        if (list_empty(&m->small_win))
                return;
        list_del_init(&m->small_win);
        *m->group->open = true;
        *wakeup = true;
}

static void tipc_group_decr_active(struct tipc_group *grp,
                                   struct tipc_member *m)
{
        if (m->state == MBR_ACTIVE || m->state == MBR_RECLAIMING ||
            m->state == MBR_REMITTED)
                grp->active_cnt--;
}

static int tipc_group_rcvbuf_limit(struct tipc_group *grp)
{
        int max_active, active_pool, idle_pool;
        int mcnt = grp->member_cnt + 1;

        /* Limit simultaneous reception from other members */
        max_active = min(mcnt / 8, 64);
        max_active = max(max_active, 16);
        grp->max_active = max_active;

        /* Reserve blocks for active and idle members */
        active_pool = max_active * ADV_ACTIVE;
        idle_pool = (mcnt - max_active) * ADV_IDLE;

        /* Scale to bytes, considering worst-case truesize/msgsize ratio */
        return (active_pool + idle_pool) * FLOWCTL_BLK_SZ * 4;
}

u16 tipc_group_bc_snd_nxt(struct tipc_group *grp)
{
        return grp->bc_snd_nxt;
}

static bool tipc_group_is_receiver(struct tipc_member *m)
{
        return m && m->state != MBR_JOINING && m->state != MBR_LEAVING;
}

static bool tipc_group_is_sender(struct tipc_member *m)
{
        return m && m->state != MBR_JOINING && m->state != MBR_PUBLISHED;
}

u32 tipc_group_exclude(struct tipc_group *grp)
{
        if (!grp->loopback)
                return grp->portid;
        return 0;
}

struct tipc_group *tipc_group_create(struct net *net, u32 portid,
                                     struct tipc_group_req *mreq,
                                     bool *group_is_open)
{
        u32 filter = TIPC_SUB_PORTS | TIPC_SUB_NO_STATUS;
        bool global = mreq->scope != TIPC_NODE_SCOPE;
        struct tipc_group *grp;
        u32 type = mreq->type;

        grp = kzalloc(sizeof(*grp), GFP_ATOMIC);
        if (!grp)
                return NULL;
        tipc_nlist_init(&grp->dests, tipc_own_addr(net));
        INIT_LIST_HEAD(&grp->small_win);
        INIT_LIST_HEAD(&grp->active);
        INIT_LIST_HEAD(&grp->pending);
        grp->members = RB_ROOT;
        grp->net = net;
        grp->portid = portid;
        grp->type = type;
        grp->instance = mreq->instance;
        grp->scope = mreq->scope;
        grp->loopback = mreq->flags & TIPC_GROUP_LOOPBACK;
        grp->events = mreq->flags & TIPC_GROUP_MEMBER_EVTS;
        grp->open = group_is_open;
        *grp->open = false;
        filter |= global ? TIPC_SUB_CLUSTER_SCOPE : TIPC_SUB_NODE_SCOPE;
        if (tipc_topsrv_kern_subscr(net, portid, type, 0, ~0,
                                    filter, &grp->subid))
                return grp;
        kfree(grp);
        return NULL;
}

void tipc_group_join(struct net *net, struct tipc_group *grp, int *sk_rcvbuf)
{
        struct rb_root *tree = &grp->members;
        struct tipc_member *m, *tmp;
        struct sk_buff_head xmitq;

        __skb_queue_head_init(&xmitq);
        rbtree_postorder_for_each_entry_safe(m, tmp, tree, tree_node) {
                tipc_group_proto_xmit(grp, m, GRP_JOIN_MSG, &xmitq);
                tipc_group_update_member(m, 0);
        }
        tipc_node_distr_xmit(net, &xmitq);
        *sk_rcvbuf = tipc_group_rcvbuf_limit(grp);
}

void tipc_group_delete(struct net *net, struct tipc_group *grp)
{
        struct rb_root *tree = &grp->members;
        struct tipc_member *m, *tmp;
        struct sk_buff_head xmitq;

        __skb_queue_head_init(&xmitq);

        rbtree_postorder_for_each_entry_safe(m, tmp, tree, tree_node) {
                tipc_group_proto_xmit(grp, m, GRP_LEAVE_MSG, &xmitq);
                __skb_queue_purge(&m->deferredq);
                list_del(&m->list);
                kfree(m);
        }
        tipc_node_distr_xmit(net, &xmitq);
        tipc_nlist_purge(&grp->dests);
        tipc_topsrv_kern_unsubscr(net, grp->subid);
        kfree(grp);
}

static struct tipc_member *tipc_group_find_member(struct tipc_group *grp,
                                                  u32 node, u32 port)
{
        struct rb_node *n = grp->members.rb_node;
        u64 nkey, key = (u64)node << 32 | port;
        struct tipc_member *m;

        while (n) {
                m = container_of(n, struct tipc_member, tree_node);
                nkey = (u64)m->node << 32 | m->port;
                if (key < nkey)
                        n = n->rb_left;
                else if (key > nkey)
                        n = n->rb_right;
                else
                        return m;
        }
        return NULL;
}

static struct tipc_member *tipc_group_find_dest(struct tipc_group *grp,
                                                u32 node, u32 port)
{
        struct tipc_member *m;

        m = tipc_group_find_member(grp, node, port);
        if (m && tipc_group_is_receiver(m))
                return m;
        return NULL;
}

static struct tipc_member *tipc_group_find_node(struct tipc_group *grp,
                                                u32 node)
{
        struct tipc_member *m;
        struct rb_node *n;

        for (n = rb_first(&grp->members); n; n = rb_next(n)) {
                m = container_of(n, struct tipc_member, tree_node);
                if (m->node == node)
                        return m;
        }
        return NULL;
}

static int tipc_group_add_to_tree(struct tipc_group *grp,
                                  struct tipc_member *m)
{
        u64 nkey, key = (u64)m->node << 32 | m->port;
        struct rb_node **n, *parent = NULL;
        struct tipc_member *tmp;

        n = &grp->members.rb_node;
        while (*n) {
                tmp = container_of(*n, struct tipc_member, tree_node);
                parent = *n;
                tmp = container_of(parent, struct tipc_member, tree_node);
                nkey = (u64)tmp->node << 32 | tmp->port;
                if (key < nkey)
                        n = &(*n)->rb_left;
                else if (key > nkey)
                        n = &(*n)->rb_right;
                else
                        return -EEXIST;
        }
        rb_link_node(&m->tree_node, parent, n);
        rb_insert_color(&m->tree_node, &grp->members);
        return 0;
}

static struct tipc_member *tipc_group_create_member(struct tipc_group *grp,
                                                    u32 node, u32 port,
                                                    u32 instance, int state)
{
        struct tipc_member *m;
        int ret;

        m = kzalloc(sizeof(*m), GFP_ATOMIC);
        if (!m)
                return NULL;
        INIT_LIST_HEAD(&m->list);
        INIT_LIST_HEAD(&m->small_win);
        __skb_queue_head_init(&m->deferredq);
        m->group = grp;
        m->node = node;
        m->port = port;
        m->instance = instance;
        m->bc_acked = grp->bc_snd_nxt - 1;
        ret = tipc_group_add_to_tree(grp, m);
        if (ret < 0) {
                kfree(m);
                return NULL;
        }
        grp->member_cnt++;
        tipc_nlist_add(&grp->dests, m->node);
        m->state = state;
        return m;
}

void tipc_group_add_member(struct tipc_group *grp, u32 node,
                           u32 port, u32 instance)
{
        tipc_group_create_member(grp, node, port, instance, MBR_PUBLISHED);
}

static void tipc_group_delete_member(struct tipc_group *grp,
                                     struct tipc_member *m)
{
        rb_erase(&m->tree_node, &grp->members);
        grp->member_cnt--;

        /* Check if we were waiting for replicast ack from this member */
        if (grp->bc_ackers && less(m->bc_acked, grp->bc_snd_nxt - 1))
                grp->bc_ackers--;

        list_del_init(&m->list);
        list_del_init(&m->small_win);
        tipc_group_decr_active(grp, m);

        /* If last member on a node, remove node from dest list */
        if (!tipc_group_find_node(grp, m->node))
                tipc_nlist_del(&grp->dests, m->node);

        kfree(m);
}

struct tipc_nlist *tipc_group_dests(struct tipc_group *grp)
{
        return &grp->dests;
}

void tipc_group_self(struct tipc_group *grp, struct tipc_service_range *seq,
                     int *scope)
{
        seq->type = grp->type;
        seq->lower = grp->instance;
        seq->upper = grp->instance;
        *scope = grp->scope;
}

void tipc_group_update_member(struct tipc_member *m, int len)
{
        struct tipc_group *grp = m->group;
        struct tipc_member *_m, *tmp;

        if (!tipc_group_is_receiver(m))
                return;

        m->window -= len;

        if (m->window >= ADV_IDLE)
                return;

        list_del_init(&m->small_win);

        /* Sort member into small_window members' list */
        list_for_each_entry_safe(_m, tmp, &grp->small_win, small_win) {
                if (_m->window > m->window)
                        break;
        }
        list_add_tail(&m->small_win, &_m->small_win);
}

void tipc_group_update_bc_members(struct tipc_group *grp, int len, bool ack)
{
        u16 prev = grp->bc_snd_nxt - 1;
        struct tipc_member *m;
        struct rb_node *n;
        u16 ackers = 0;

        for (n = rb_first(&grp->members); n; n = rb_next(n)) {
                m = container_of(n, struct tipc_member, tree_node);
                if (tipc_group_is_receiver(m)) {
                        tipc_group_update_member(m, len);
                        m->bc_acked = prev;
                        ackers++;
                }
        }

        /* Mark number of acknowledges to expect, if any */
        if (ack)
                grp->bc_ackers = ackers;
        grp->bc_snd_nxt++;
}

bool tipc_group_cong(struct tipc_group *grp, u32 dnode, u32 dport,
                     int len, struct tipc_member **mbr)
{
        struct sk_buff_head xmitq;
        struct tipc_member *m;
        int adv, state;

        m = tipc_group_find_dest(grp, dnode, dport);
        if (!tipc_group_is_receiver(m)) {
                *mbr = NULL;
                return false;
        }
        *mbr = m;

        if (m->window >= len)
                return false;

        *grp->open = false;

        /* If not fully advertised, do it now to prevent mutual blocking */
        adv = m->advertised;
        state = m->state;
        if (state == MBR_JOINED && adv == ADV_IDLE)
                return true;
        if (state == MBR_ACTIVE && adv == ADV_ACTIVE)
                return true;
        if (state == MBR_PENDING && adv == ADV_IDLE)
                return true;
        __skb_queue_head_init(&xmitq);
        tipc_group_proto_xmit(grp, m, GRP_ADV_MSG, &xmitq);
        tipc_node_distr_xmit(grp->net, &xmitq);
        return true;
}

bool tipc_group_bc_cong(struct tipc_group *grp, int len)
{
        struct tipc_member *m = NULL;

        /* If prev bcast was replicast, reject until all receivers have acked */
        if (grp->bc_ackers) {
                *grp->open = false;
                return true;
        }
        if (list_empty(&grp->small_win))
                return false;

        m = list_first_entry(&grp->small_win, struct tipc_member, small_win);
        if (m->window >= len)
                return false;

        return tipc_group_cong(grp, m->node, m->port, len, &m);
}

/* tipc_group_sort_msg() - sort msg into queue by bcast sequence number
 */
static void tipc_group_sort_msg(struct sk_buff *skb, struct sk_buff_head *defq)
{
        struct tipc_msg *_hdr, *hdr = buf_msg(skb);
        u16 bc_seqno = msg_grp_bc_seqno(hdr);
        struct sk_buff *_skb, *tmp;
        int mtyp = msg_type(hdr);

        /* Bcast/mcast may be bypassed by ucast or other bcast, - sort it in */
        if (mtyp == TIPC_GRP_BCAST_MSG || mtyp == TIPC_GRP_MCAST_MSG) {
                skb_queue_walk_safe(defq, _skb, tmp) {
                        _hdr = buf_msg(_skb);
                        if (!less(bc_seqno, msg_grp_bc_seqno(_hdr)))
                                continue;
                        __skb_queue_before(defq, _skb, skb);
                        return;
                }
                /* Bcast was not bypassed, - add to tail */
        }
        /* Unicasts are never bypassed, - always add to tail */
        __skb_queue_tail(defq, skb);
}

/* tipc_group_filter_msg() - determine if we should accept arriving message
 */
void tipc_group_filter_msg(struct tipc_group *grp, struct sk_buff_head *inputq,
                           struct sk_buff_head *xmitq)
{
        struct sk_buff *skb = __skb_dequeue(inputq);
        bool ack, deliver, update, leave = false;
        struct sk_buff_head *defq;
        struct tipc_member *m;
        struct tipc_msg *hdr;
        u32 node, port;
        int mtyp, blks;

        if (!skb)
                return;

        hdr = buf_msg(skb);
        node =  msg_orignode(hdr);
        port = msg_origport(hdr);

        if (!msg_in_group(hdr))
                goto drop;

        m = tipc_group_find_member(grp, node, port);
        if (!tipc_group_is_sender(m))
                goto drop;

        if (less(msg_grp_bc_seqno(hdr), m->bc_rcv_nxt))
                goto drop;

        TIPC_SKB_CB(skb)->orig_member = m->instance;
        defq = &m->deferredq;
        tipc_group_sort_msg(skb, defq);

        while ((skb = skb_peek(defq))) {
                hdr = buf_msg(skb);
                mtyp = msg_type(hdr);
                blks = msg_blocks(hdr);
                deliver = true;
                ack = false;
                update = false;

                if (more(msg_grp_bc_seqno(hdr), m->bc_rcv_nxt))
                        break;

                /* Decide what to do with message */
                switch (mtyp) {
                case TIPC_GRP_MCAST_MSG:
                        if (msg_nameinst(hdr) != grp->instance) {
                                update = true;
                                deliver = false;
                        }
                        fallthrough;
                case TIPC_GRP_BCAST_MSG:
                        m->bc_rcv_nxt++;
                        ack = msg_grp_bc_ack_req(hdr);
                        break;
                case TIPC_GRP_UCAST_MSG:
                        break;
                case TIPC_GRP_MEMBER_EVT:
                        if (m->state == MBR_LEAVING)
                                leave = true;
                        if (!grp->events)
                                deliver = false;
                        break;
                default:
                        break;
                }

                /* Execute decisions */
                __skb_dequeue(defq);
                if (deliver)
                        __skb_queue_tail(inputq, skb);
                else
                        kfree_skb(skb);

                if (ack)
                        tipc_group_proto_xmit(grp, m, GRP_ACK_MSG, xmitq);

                if (leave) {
                        __skb_queue_purge(defq);
                        tipc_group_delete_member(grp, m);
                        break;
                }
                if (!update)
                        continue;

                tipc_group_update_rcv_win(grp, blks, node, port, xmitq);
        }
        return;
drop:
        kfree_skb(skb);
}

void tipc_group_update_rcv_win(struct tipc_group *grp, int blks, u32 node,
                               u32 port, struct sk_buff_head *xmitq)
{
        struct list_head *active = &grp->active;
        int max_active = grp->max_active;
        int reclaim_limit = max_active * 3 / 4;
        int active_cnt = grp->active_cnt;
        struct tipc_member *m, *rm, *pm;

        m = tipc_group_find_member(grp, node, port);
        if (!m)
                return;

        m->advertised -= blks;

        switch (m->state) {
        case MBR_JOINED:
                /* First, decide if member can go active */
                if (active_cnt <= max_active) {
                        m->state = MBR_ACTIVE;
                        list_add_tail(&m->list, active);
                        grp->active_cnt++;
                        tipc_group_proto_xmit(grp, m, GRP_ADV_MSG, xmitq);
                } else {
                        m->state = MBR_PENDING;
                        list_add_tail(&m->list, &grp->pending);
                }

                if (active_cnt < reclaim_limit)
                        break;

                /* Reclaim from oldest active member, if possible */
                if (!list_empty(active)) {
                        rm = list_first_entry(active, struct tipc_member, list);
                        rm->state = MBR_RECLAIMING;
                        list_del_init(&rm->list);
                        tipc_group_proto_xmit(grp, rm, GRP_RECLAIM_MSG, xmitq);
                        break;
                }
                /* Nobody to reclaim from; - revert oldest pending to JOINED */
                pm = list_first_entry(&grp->pending, struct tipc_member, list);
                list_del_init(&pm->list);
                pm->state = MBR_JOINED;
                tipc_group_proto_xmit(grp, pm, GRP_ADV_MSG, xmitq);
                break;
        case MBR_ACTIVE:
                if (!list_is_last(&m->list, &grp->active))
                        list_move_tail(&m->list, &grp->active);
                if (m->advertised > (ADV_ACTIVE * 3 / 4))
                        break;
                tipc_group_proto_xmit(grp, m, GRP_ADV_MSG, xmitq);
                break;
        case MBR_REMITTED:
                if (m->advertised > ADV_IDLE)
                        break;
                m->state = MBR_JOINED;
                grp->active_cnt--;
                if (m->advertised < ADV_IDLE) {
                        pr_warn_ratelimited("Rcv unexpected msg after REMIT\n");
                        tipc_group_proto_xmit(grp, m, GRP_ADV_MSG, xmitq);
                }

                if (list_empty(&grp->pending))
                        return;

                /* Set oldest pending member to active and advertise */
                pm = list_first_entry(&grp->pending, struct tipc_member, list);
                pm->state = MBR_ACTIVE;
                list_move_tail(&pm->list, &grp->active);
                grp->active_cnt++;
                tipc_group_proto_xmit(grp, pm, GRP_ADV_MSG, xmitq);
                break;
        case MBR_RECLAIMING:
        case MBR_JOINING:
        case MBR_LEAVING:
        default:
                break;
        }
}

static void tipc_group_create_event(struct tipc_group *grp,
                                    struct tipc_member *m,
                                    u32 event, u16 seqno,
                                    struct sk_buff_head *inputq)
{        u32 dnode = tipc_own_addr(grp->net);
        struct tipc_event evt;
        struct sk_buff *skb;
        struct tipc_msg *hdr;

        memset(&evt, 0, sizeof(evt));
        evt.event = event;
        evt.found_lower = m->instance;
        evt.found_upper = m->instance;
        evt.port.ref = m->port;
        evt.port.node = m->node;
        evt.s.seq.type = grp->type;
        evt.s.seq.lower = m->instance;
        evt.s.seq.upper = m->instance;

        skb = tipc_msg_create(TIPC_CRITICAL_IMPORTANCE, TIPC_GRP_MEMBER_EVT,
                              GROUP_H_SIZE, sizeof(evt), dnode, m->node,
                              grp->portid, m->port, 0);
        if (!skb)
                return;

        hdr = buf_msg(skb);
        msg_set_nametype(hdr, grp->type);
        msg_set_grp_evt(hdr, event);
        msg_set_dest_droppable(hdr, true);
        msg_set_grp_bc_seqno(hdr, seqno);
        memcpy(msg_data(hdr), &evt, sizeof(evt));
        TIPC_SKB_CB(skb)->orig_member = m->instance;
        __skb_queue_tail(inputq, skb);
}

static void tipc_group_proto_xmit(struct tipc_group *grp, struct tipc_member *m,
                                  int mtyp, struct sk_buff_head *xmitq)
{
        struct tipc_msg *hdr;
        struct sk_buff *skb;
        int adv = 0;

        skb = tipc_msg_create(GROUP_PROTOCOL, mtyp, INT_H_SIZE, 0,
                              m->node, tipc_own_addr(grp->net),
                              m->port, grp->portid, 0);
        if (!skb)
                return;

        if (m->state == MBR_ACTIVE)
                adv = ADV_ACTIVE - m->advertised;
        else if (m->state == MBR_JOINED || m->state == MBR_PENDING)
                adv = ADV_IDLE - m->advertised;

        hdr = buf_msg(skb);

        if (mtyp == GRP_JOIN_MSG) {
                msg_set_grp_bc_syncpt(hdr, grp->bc_snd_nxt);
                msg_set_adv_win(hdr, adv);
                m->advertised += adv;
        } else if (mtyp == GRP_LEAVE_MSG) {
                msg_set_grp_bc_syncpt(hdr, grp->bc_snd_nxt);
        } else if (mtyp == GRP_ADV_MSG) {
                msg_set_adv_win(hdr, adv);
                m->advertised += adv;
        } else if (mtyp == GRP_ACK_MSG) {
                msg_set_grp_bc_acked(hdr, m->bc_rcv_nxt);
        } else if (mtyp == GRP_REMIT_MSG) {
                msg_set_grp_remitted(hdr, m->window);
        }
        msg_set_dest_droppable(hdr, true);
        __skb_queue_tail(xmitq, skb);
}

void tipc_group_proto_rcv(struct tipc_group *grp, bool *usr_wakeup,
                          struct tipc_msg *hdr, struct sk_buff_head *inputq,
                          struct sk_buff_head *xmitq)
{
        u32 node = msg_orignode(hdr);
        u32 port = msg_origport(hdr);
        struct tipc_member *m, *pm;
        u16 remitted, in_flight;

        if (!grp)
                return;

        if (grp->scope == TIPC_NODE_SCOPE && node != tipc_own_addr(grp->net))
                return;

        m = tipc_group_find_member(grp, node, port);

        switch (msg_type(hdr)) {
        case GRP_JOIN_MSG:
                if (!m)
                        m = tipc_group_create_member(grp, node, port,
                                                     0, MBR_JOINING);
                if (!m)
                        return;
                m->bc_syncpt = msg_grp_bc_syncpt(hdr);
                m->bc_rcv_nxt = m->bc_syncpt;
                m->window += msg_adv_win(hdr);

                /* Wait until PUBLISH event is received if necessary */
                if (m->state != MBR_PUBLISHED)
                        return;

                /* Member can be taken into service */
                m->state = MBR_JOINED;
                tipc_group_open(m, usr_wakeup);
                tipc_group_update_member(m, 0);
                tipc_group_proto_xmit(grp, m, GRP_ADV_MSG, xmitq);
                tipc_group_create_event(grp, m, TIPC_PUBLISHED,
                                        m->bc_syncpt, inputq);
                return;
        case GRP_LEAVE_MSG:
                if (!m)
                        return;
                m->bc_syncpt = msg_grp_bc_syncpt(hdr);
                list_del_init(&m->list);
                tipc_group_open(m, usr_wakeup);
                tipc_group_decr_active(grp, m);
                m->state = MBR_LEAVING;
                tipc_group_create_event(grp, m, TIPC_WITHDRAWN,
                                        m->bc_syncpt, inputq);
                return;
        case GRP_ADV_MSG:
                if (!m)
                        return;
                m->window += msg_adv_win(hdr);
                tipc_group_open(m, usr_wakeup);
                return;
        case GRP_ACK_MSG:
                if (!m)
                        return;
                m->bc_acked = msg_grp_bc_acked(hdr);
                if (--grp->bc_ackers)
                        return;
                list_del_init(&m->small_win);
                *m->group->open = true;
                *usr_wakeup = true;
                tipc_group_update_member(m, 0);
                return;
        case GRP_RECLAIM_MSG:
                if (!m)
                        return;
                tipc_group_proto_xmit(grp, m, GRP_REMIT_MSG, xmitq);
                m->window = ADV_IDLE;
                tipc_group_open(m, usr_wakeup);
                return;
        case GRP_REMIT_MSG:
                if (!m || m->state != MBR_RECLAIMING)
                        return;

                remitted = msg_grp_remitted(hdr);

                /* Messages preceding the REMIT still in receive queue */
                if (m->advertised > remitted) {
                        m->state = MBR_REMITTED;
                        in_flight = m->advertised - remitted;
                        m->advertised = ADV_IDLE + in_flight;
                        return;
                }
                /* This should never happen */
                if (m->advertised < remitted)
                        pr_warn_ratelimited("Unexpected REMIT msg\n");

                /* All messages preceding the REMIT have been read */
                m->state = MBR_JOINED;
                grp->active_cnt--;
                m->advertised = ADV_IDLE;

                /* Set oldest pending member to active and advertise */
                if (list_empty(&grp->pending))
                        return;
                pm = list_first_entry(&grp->pending, struct tipc_member, list);
                pm->state = MBR_ACTIVE;
                list_move_tail(&pm->list, &grp->active);
                grp->active_cnt++;
                if (pm->advertised <= (ADV_ACTIVE * 3 / 4))
                        tipc_group_proto_xmit(grp, pm, GRP_ADV_MSG, xmitq);
                return;
        default:
                pr_warn("Received unknown GROUP_PROTO message\n");
        }
}

/* tipc_group_member_evt() - receive and handle a member up/down event
 */
void tipc_group_member_evt(struct tipc_group *grp,
                           bool *usr_wakeup,
                           int *sk_rcvbuf,
                           struct tipc_msg *hdr,
                           struct sk_buff_head *inputq,
                           struct sk_buff_head *xmitq)
{
        struct tipc_event *evt = (void *)msg_data(hdr);
        u32 instance = evt->found_lower;
        u32 node = evt->port.node;
        u32 port = evt->port.ref;
        int event = evt->event;
        struct tipc_member *m;
        struct net *net;
        u32 self;

        if (!grp)
                return;

        net = grp->net;
        self = tipc_own_addr(net);
        if (!grp->loopback && node == self && port == grp->portid)
                return;

        m = tipc_group_find_member(grp, node, port);

        switch (event) {
        case TIPC_PUBLISHED:
                /* Send and wait for arrival of JOIN message if necessary */
                if (!m) {
                        m = tipc_group_create_member(grp, node, port, instance,
                                                     MBR_PUBLISHED);
                        if (!m)
                                break;
                        tipc_group_update_member(m, 0);
                        tipc_group_proto_xmit(grp, m, GRP_JOIN_MSG, xmitq);
                        break;
                }

                if (m->state != MBR_JOINING)
                        break;

                /* Member can be taken into service */
                m->instance = instance;
                m->state = MBR_JOINED;
                tipc_group_open(m, usr_wakeup);
                tipc_group_update_member(m, 0);
                tipc_group_proto_xmit(grp, m, GRP_JOIN_MSG, xmitq);
                tipc_group_create_event(grp, m, TIPC_PUBLISHED,
                                        m->bc_syncpt, inputq);
                break;
        case TIPC_WITHDRAWN:
                if (!m)
                        break;

                tipc_group_decr_active(grp, m);
                m->state = MBR_LEAVING;
                list_del_init(&m->list);
                tipc_group_open(m, usr_wakeup);

                /* Only send event if no LEAVE message can be expected */
                if (!tipc_node_is_up(net, node))
                        tipc_group_create_event(grp, m, TIPC_WITHDRAWN,
                                                m->bc_rcv_nxt, inputq);
                break;
        default:
                break;
        }
        *sk_rcvbuf = tipc_group_rcvbuf_limit(grp);
}

int tipc_group_fill_sock_diag(struct tipc_group *grp, struct sk_buff *skb)
{
        struct nlattr *group = nla_nest_start_noflag(skb, TIPC_NLA_SOCK_GROUP);

        if (!group)
                return -EMSGSIZE;

        if (nla_put_u32(skb, TIPC_NLA_SOCK_GROUP_ID,
                        grp->type) ||
            nla_put_u32(skb, TIPC_NLA_SOCK_GROUP_INSTANCE,
                        grp->instance) ||
            nla_put_u32(skb, TIPC_NLA_SOCK_GROUP_BC_SEND_NEXT,
                        grp->bc_snd_nxt))
                goto group_msg_cancel;

        if (grp->scope == TIPC_NODE_SCOPE)
                if (nla_put_flag(skb, TIPC_NLA_SOCK_GROUP_NODE_SCOPE))
                        goto group_msg_cancel;

        if (grp->scope == TIPC_CLUSTER_SCOPE)
                if (nla_put_flag(skb, TIPC_NLA_SOCK_GROUP_CLUSTER_SCOPE))
                        goto group_msg_cancel;

        if (*grp->open)
                if (nla_put_flag(skb, TIPC_NLA_SOCK_GROUP_OPEN))
                        goto group_msg_cancel;

        nla_nest_end(skb, group);
        return 0;

group_msg_cancel:
        nla_nest_cancel(skb, group);
        return -1;
}

































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    7 
    7 


















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   52 






















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































  223 




  223 
  223 
  223 

  223 





























    2 



    2 
    2 
    2 

    2 











  221 



  221 
  221 
  220 
  221 


  220 

































































































































































































































































































































































































































































































































































































































































    8 


    8 

    8 





    7 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
/*
 *  kernel/cpuset.c
 *
 *  Processor and Memory placement constraints for sets of tasks.
 *
 *  Copyright (C) 2003 BULL SA.
 *  Copyright (C) 2004-2007 Silicon Graphics, Inc.
 *  Copyright (C) 2006 Google, Inc
 *
 *  Portions derived from Patrick Mochel's sysfs code.
 *  sysfs is Copyright (c) 2001-3 Patrick Mochel
 *
 *  2003-10-10 Written by Simon Derr.
 *  2003-10-22 Updates by Stephen Hemminger.
 *  2004 May-July Rework by Paul Jackson.
 *  2006 Rework by Paul Menage to use generic cgroups
 *  2008 Rework of the scheduler domains and CPU hotplug handling
 *       by Max Krasnyansky
 *
 *  This file is subject to the terms and conditions of the GNU General Public
 *  License.  See the file COPYING in the main directory of the Linux
 *  distribution for more details.
 */
#include "cpuset-internal.h"

#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/mempolicy.h>
#include <linux/mm.h>
#include <linux/memory.h>
#include <linux/export.h>
#include <linux/rcupdate.h>
#include <linux/sched.h>
#include <linux/sched/deadline.h>
#include <linux/sched/mm.h>
#include <linux/sched/task.h>
#include <linux/security.h>
#include <linux/oom.h>
#include <linux/sched/isolation.h>
#include <linux/wait.h>
#include <linux/workqueue.h>

DEFINE_STATIC_KEY_FALSE(cpusets_pre_enable_key);
DEFINE_STATIC_KEY_FALSE(cpusets_enabled_key);

/*
 * There could be abnormal cpuset configurations for cpu or memory
 * node binding, add this key to provide a quick low-cost judgment
 * of the situation.
 */
DEFINE_STATIC_KEY_FALSE(cpusets_insane_config_key);

static const char * const perr_strings[] = {
        [PERR_INVCPUS]   = "Invalid cpu list in cpuset.cpus.exclusive",
        [PERR_INVPARENT] = "Parent is an invalid partition root",
        [PERR_NOTPART]   = "Parent is not a partition root",
        [PERR_NOTEXCL]   = "Cpu list in cpuset.cpus not exclusive",
        [PERR_NOCPUS]    = "Parent unable to distribute cpu downstream",
        [PERR_HOTPLUG]   = "No cpu available due to hotplug",
        [PERR_CPUSEMPTY] = "cpuset.cpus and cpuset.cpus.exclusive are empty",
        [PERR_HKEEPING]  = "partition config conflicts with housekeeping setup",
        [PERR_ACCESS]    = "Enable partition not permitted",
        [PERR_REMOTE]    = "Have remote partition underneath",
};

/*
 * For local partitions, update to subpartitions_cpus & isolated_cpus is done
 * in update_parent_effective_cpumask(). For remote partitions, it is done in
 * the remote_partition_*() and remote_cpus_update() helpers.
 */
/*
 * Exclusive CPUs distributed out to local or remote sub-partitions of
 * top_cpuset
 */
static cpumask_var_t        subpartitions_cpus;

/*
 * Exclusive CPUs in isolated partitions
 */
static cpumask_var_t        isolated_cpus;

/*
 * Housekeeping (HK_TYPE_DOMAIN) CPUs at boot
 */
static cpumask_var_t        boot_hk_cpus;
static bool                have_boot_isolcpus;

/* List of remote partition root children */
static struct list_head remote_children;

/*
 * A flag to force sched domain rebuild at the end of an operation.
 * It can be set in
 *  - update_partition_sd_lb()
 *  - update_cpumasks_hier()
 *  - cpuset_update_flag()
 *  - cpuset_hotplug_update_tasks()
 *  - cpuset_handle_hotplug()
 *
 * Protected by cpuset_mutex (with cpus_read_lock held) or cpus_write_lock.
 *
 * Note that update_relax_domain_level() in cpuset-v1.c can still call
 * rebuild_sched_domains_locked() directly without using this flag.
 */
static bool force_sd_rebuild;

/*
 * Partition root states:
 *
 *   0 - member (not a partition root)
 *   1 - partition root
 *   2 - partition root without load balancing (isolated)
 *  -1 - invalid partition root
 *  -2 - invalid isolated partition root
 *
 *  There are 2 types of partitions - local or remote. Local partitions are
 *  those whose parents are partition root themselves. Setting of
 *  cpuset.cpus.exclusive are optional in setting up local partitions.
 *  Remote partitions are those whose parents are not partition roots. Passing
 *  down exclusive CPUs by setting cpuset.cpus.exclusive along its ancestor
 *  nodes are mandatory in creating a remote partition.
 *
 *  For simplicity, a local partition can be created under a local or remote
 *  partition but a remote partition cannot have any partition root in its
 *  ancestor chain except the cgroup root.
 */
#define PRS_MEMBER                0
#define PRS_ROOT                1
#define PRS_ISOLATED                2
#define PRS_INVALID_ROOT        -1
#define PRS_INVALID_ISOLATED        -2

static inline bool is_prs_invalid(int prs_state)
{
        return prs_state < 0;
}

/*
 * Temporary cpumasks for working with partitions that are passed among
 * functions to avoid memory allocation in inner functions.
 */
struct tmpmasks {
        cpumask_var_t addmask, delmask;        /* For partition root */
        cpumask_var_t new_cpus;                /* For update_cpumasks_hier() */
};

void inc_dl_tasks_cs(struct task_struct *p)
{
        struct cpuset *cs = task_cs(p);

        cs->nr_deadline_tasks++;
}

void dec_dl_tasks_cs(struct task_struct *p)
{
        struct cpuset *cs = task_cs(p);

        cs->nr_deadline_tasks--;
}

static inline int is_partition_valid(const struct cpuset *cs)
{
        return cs->partition_root_state > 0;
}

static inline int is_partition_invalid(const struct cpuset *cs)
{
        return cs->partition_root_state < 0;
}

/*
 * Callers should hold callback_lock to modify partition_root_state.
 */
static inline void make_partition_invalid(struct cpuset *cs)
{
        if (cs->partition_root_state > 0)
                cs->partition_root_state = -cs->partition_root_state;
}

/*
 * Send notification event of whenever partition_root_state changes.
 */
static inline void notify_partition_change(struct cpuset *cs, int old_prs)
{
        if (old_prs == cs->partition_root_state)
                return;
        cgroup_file_notify(&cs->partition_file);

        /* Reset prs_err if not invalid */
        if (is_partition_valid(cs))
                WRITE_ONCE(cs->prs_err, PERR_NONE);
}

static struct cpuset top_cpuset = {
        .flags = BIT(CS_ONLINE) | BIT(CS_CPU_EXCLUSIVE) |
                 BIT(CS_MEM_EXCLUSIVE) | BIT(CS_SCHED_LOAD_BALANCE),
        .partition_root_state = PRS_ROOT,
        .relax_domain_level = -1,
        .remote_sibling = LIST_HEAD_INIT(top_cpuset.remote_sibling),
};

/*
 * There are two global locks guarding cpuset structures - cpuset_mutex and
 * callback_lock. The cpuset code uses only cpuset_mutex. Other kernel
 * subsystems can use cpuset_lock()/cpuset_unlock() to prevent change to cpuset
 * structures. Note that cpuset_mutex needs to be a mutex as it is used in
 * paths that rely on priority inheritance (e.g. scheduler - on RT) for
 * correctness.
 *
 * A task must hold both locks to modify cpusets.  If a task holds
 * cpuset_mutex, it blocks others, ensuring that it is the only task able to
 * also acquire callback_lock and be able to modify cpusets.  It can perform
 * various checks on the cpuset structure first, knowing nothing will change.
 * It can also allocate memory while just holding cpuset_mutex.  While it is
 * performing these checks, various callback routines can briefly acquire
 * callback_lock to query cpusets.  Once it is ready to make the changes, it
 * takes callback_lock, blocking everyone else.
 *
 * Calls to the kernel memory allocator can not be made while holding
 * callback_lock, as that would risk double tripping on callback_lock
 * from one of the callbacks into the cpuset code from within
 * __alloc_pages().
 *
 * If a task is only holding callback_lock, then it has read-only
 * access to cpusets.
 *
 * Now, the task_struct fields mems_allowed and mempolicy may be changed
 * by other task, we use alloc_lock in the task_struct fields to protect
 * them.
 *
 * The cpuset_common_seq_show() handlers only hold callback_lock across
 * small pieces of code, such as when reading out possibly multi-word
 * cpumasks and nodemasks.
 */

static DEFINE_MUTEX(cpuset_mutex);

void cpuset_lock(void)
{
        mutex_lock(&cpuset_mutex);
}

void cpuset_unlock(void)
{
        mutex_unlock(&cpuset_mutex);
}

static DEFINE_SPINLOCK(callback_lock);

void cpuset_callback_lock_irq(void)
{
        spin_lock_irq(&callback_lock);
}

void cpuset_callback_unlock_irq(void)
{
        spin_unlock_irq(&callback_lock);
}

static struct workqueue_struct *cpuset_migrate_mm_wq;

static DECLARE_WAIT_QUEUE_HEAD(cpuset_attach_wq);

static inline void check_insane_mems_config(nodemask_t *nodes)
{
        if (!cpusets_insane_config() &&
                movable_only_nodes(nodes)) {
                static_branch_enable(&cpusets_insane_config_key);
                pr_info("Unsupported (movable nodes only) cpuset configuration detected (nmask=%*pbl)!\n"
                        "Cpuset allocations might fail even with a lot of memory available.\n",
                        nodemask_pr_args(nodes));
        }
}

/*
 * decrease cs->attach_in_progress.
 * wake_up cpuset_attach_wq if cs->attach_in_progress==0.
 */
static inline void dec_attach_in_progress_locked(struct cpuset *cs)
{
        lockdep_assert_held(&cpuset_mutex);

        cs->attach_in_progress--;
        if (!cs->attach_in_progress)
                wake_up(&cpuset_attach_wq);
}

static inline void dec_attach_in_progress(struct cpuset *cs)
{
        mutex_lock(&cpuset_mutex);
        dec_attach_in_progress_locked(cs);
        mutex_unlock(&cpuset_mutex);
}

static inline bool cpuset_v2(void)
{
        return !IS_ENABLED(CONFIG_CPUSETS_V1) ||
                cgroup_subsys_on_dfl(cpuset_cgrp_subsys);
}

/*
 * Cgroup v2 behavior is used on the "cpus" and "mems" control files when
 * on default hierarchy or when the cpuset_v2_mode flag is set by mounting
 * the v1 cpuset cgroup filesystem with the "cpuset_v2_mode" mount option.
 * With v2 behavior, "cpus" and "mems" are always what the users have
 * requested and won't be changed by hotplug events. Only the effective
 * cpus or mems will be affected.
 */
static inline bool is_in_v2_mode(void)
{
        return cpuset_v2() ||
              (cpuset_cgrp_subsys.root->flags & CGRP_ROOT_CPUSET_V2_MODE);
}

/**
 * partition_is_populated - check if partition has tasks
 * @cs: partition root to be checked
 * @excluded_child: a child cpuset to be excluded in task checking
 * Return: true if there are tasks, false otherwise
 *
 * It is assumed that @cs is a valid partition root. @excluded_child should
 * be non-NULL when this cpuset is going to become a partition itself.
 */
static inline bool partition_is_populated(struct cpuset *cs,
                                          struct cpuset *excluded_child)
{
        struct cgroup_subsys_state *css;
        struct cpuset *child;

        if (cs->css.cgroup->nr_populated_csets)
                return true;
        if (!excluded_child && !cs->nr_subparts)
                return cgroup_is_populated(cs->css.cgroup);

        rcu_read_lock();
        cpuset_for_each_child(child, css, cs) {
                if (child == excluded_child)
                        continue;
                if (is_partition_valid(child))
                        continue;
                if (cgroup_is_populated(child->css.cgroup)) {
                        rcu_read_unlock();
                        return true;
                }
        }
        rcu_read_unlock();
        return false;
}

/*
 * Return in pmask the portion of a task's cpusets's cpus_allowed that
 * are online and are capable of running the task.  If none are found,
 * walk up the cpuset hierarchy until we find one that does have some
 * appropriate cpus.
 *
 * One way or another, we guarantee to return some non-empty subset
 * of cpu_online_mask.
 *
 * Call with callback_lock or cpuset_mutex held.
 */
static void guarantee_online_cpus(struct task_struct *tsk,
                                  struct cpumask *pmask)
{
        const struct cpumask *possible_mask = task_cpu_possible_mask(tsk);
        struct cpuset *cs;

        if (WARN_ON(!cpumask_and(pmask, possible_mask, cpu_online_mask)))
                cpumask_copy(pmask, cpu_online_mask);

        rcu_read_lock();
        cs = task_cs(tsk);

        while (!cpumask_intersects(cs->effective_cpus, pmask))
                cs = parent_cs(cs);

        cpumask_and(pmask, pmask, cs->effective_cpus);
        rcu_read_unlock();
}

/*
 * Return in *pmask the portion of a cpusets's mems_allowed that
 * are online, with memory.  If none are online with memory, walk
 * up the cpuset hierarchy until we find one that does have some
 * online mems.  The top cpuset always has some mems online.
 *
 * One way or another, we guarantee to return some non-empty subset
 * of node_states[N_MEMORY].
 *
 * Call with callback_lock or cpuset_mutex held.
 */
static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask)
{
        while (!nodes_intersects(cs->effective_mems, node_states[N_MEMORY]))
                cs = parent_cs(cs);
        nodes_and(*pmask, cs->effective_mems, node_states[N_MEMORY]);
}

/**
 * alloc_cpumasks - allocate three cpumasks for cpuset
 * @cs:  the cpuset that have cpumasks to be allocated.
 * @tmp: the tmpmasks structure pointer
 * Return: 0 if successful, -ENOMEM otherwise.
 *
 * Only one of the two input arguments should be non-NULL.
 */
static inline int alloc_cpumasks(struct cpuset *cs, struct tmpmasks *tmp)
{
        cpumask_var_t *pmask1, *pmask2, *pmask3, *pmask4;

        if (cs) {
                pmask1 = &cs->cpus_allowed;
                pmask2 = &cs->effective_cpus;
                pmask3 = &cs->effective_xcpus;
                pmask4 = &cs->exclusive_cpus;
        } else {
                pmask1 = &tmp->new_cpus;
                pmask2 = &tmp->addmask;
                pmask3 = &tmp->delmask;
                pmask4 = NULL;
        }

        if (!zalloc_cpumask_var(pmask1, GFP_KERNEL))
                return -ENOMEM;

        if (!zalloc_cpumask_var(pmask2, GFP_KERNEL))
                goto free_one;

        if (!zalloc_cpumask_var(pmask3, GFP_KERNEL))
                goto free_two;

        if (pmask4 && !zalloc_cpumask_var(pmask4, GFP_KERNEL))
                goto free_three;


        return 0;

free_three:
        free_cpumask_var(*pmask3);
free_two:
        free_cpumask_var(*pmask2);
free_one:
        free_cpumask_var(*pmask1);
        return -ENOMEM;
}

/**
 * free_cpumasks - free cpumasks in a tmpmasks structure
 * @cs:  the cpuset that have cpumasks to be free.
 * @tmp: the tmpmasks structure pointer
 */
static inline void free_cpumasks(struct cpuset *cs, struct tmpmasks *tmp)
{
        if (cs) {
                free_cpumask_var(cs->cpus_allowed);
                free_cpumask_var(cs->effective_cpus);
                free_cpumask_var(cs->effective_xcpus);
                free_cpumask_var(cs->exclusive_cpus);
        }
        if (tmp) {
                free_cpumask_var(tmp->new_cpus);
                free_cpumask_var(tmp->addmask);
                free_cpumask_var(tmp->delmask);
        }
}

/**
 * alloc_trial_cpuset - allocate a trial cpuset
 * @cs: the cpuset that the trial cpuset duplicates
 */
static struct cpuset *alloc_trial_cpuset(struct cpuset *cs)
{
        struct cpuset *trial;

        trial = kmemdup(cs, sizeof(*cs), GFP_KERNEL);
        if (!trial)
                return NULL;

        if (alloc_cpumasks(trial, NULL)) {
                kfree(trial);
                return NULL;
        }

        cpumask_copy(trial->cpus_allowed, cs->cpus_allowed);
        cpumask_copy(trial->effective_cpus, cs->effective_cpus);
        cpumask_copy(trial->effective_xcpus, cs->effective_xcpus);
        cpumask_copy(trial->exclusive_cpus, cs->exclusive_cpus);
        return trial;
}

/**
 * free_cpuset - free the cpuset
 * @cs: the cpuset to be freed
 */
static inline void free_cpuset(struct cpuset *cs)
{
        free_cpumasks(cs, NULL);
        kfree(cs);
}

/* Return user specified exclusive CPUs */
static inline struct cpumask *user_xcpus(struct cpuset *cs)
{
        return cpumask_empty(cs->exclusive_cpus) ? cs->cpus_allowed
                                                 : cs->exclusive_cpus;
}

static inline bool xcpus_empty(struct cpuset *cs)
{
        return cpumask_empty(cs->cpus_allowed) &&
               cpumask_empty(cs->exclusive_cpus);
}

/*
 * cpusets_are_exclusive() - check if two cpusets are exclusive
 *
 * Return true if exclusive, false if not
 */
static inline bool cpusets_are_exclusive(struct cpuset *cs1, struct cpuset *cs2)
{
        struct cpumask *xcpus1 = user_xcpus(cs1);
        struct cpumask *xcpus2 = user_xcpus(cs2);

        if (cpumask_intersects(xcpus1, xcpus2))
                return false;
        return true;
}

/*
 * validate_change() - Used to validate that any proposed cpuset change
 *                       follows the structural rules for cpusets.
 *
 * If we replaced the flag and mask values of the current cpuset
 * (cur) with those values in the trial cpuset (trial), would
 * our various subset and exclusive rules still be valid?  Presumes
 * cpuset_mutex held.
 *
 * 'cur' is the address of an actual, in-use cpuset.  Operations
 * such as list traversal that depend on the actual address of the
 * cpuset in the list must use cur below, not trial.
 *
 * 'trial' is the address of bulk structure copy of cur, with
 * perhaps one or more of the fields cpus_allowed, mems_allowed,
 * or flags changed to new, trial values.
 *
 * Return 0 if valid, -errno if not.
 */

static int validate_change(struct cpuset *cur, struct cpuset *trial)
{
        struct cgroup_subsys_state *css;
        struct cpuset *c, *par;
        int ret = 0;

        rcu_read_lock();

        if (!is_in_v2_mode())
                ret = cpuset1_validate_change(cur, trial);
        if (ret)
                goto out;

        /* Remaining checks don't apply to root cpuset */
        if (cur == &top_cpuset)
                goto out;

        par = parent_cs(cur);

        /*
         * Cpusets with tasks - existing or newly being attached - can't
         * be changed to have empty cpus_allowed or mems_allowed.
         */
        ret = -ENOSPC;
        if ((cgroup_is_populated(cur->css.cgroup) || cur->attach_in_progress)) {
                if (!cpumask_empty(cur->cpus_allowed) &&
                    cpumask_empty(trial->cpus_allowed))
                        goto out;
                if (!nodes_empty(cur->mems_allowed) &&
                    nodes_empty(trial->mems_allowed))
                        goto out;
        }

        /*
         * We can't shrink if we won't have enough room for SCHED_DEADLINE
         * tasks. This check is not done when scheduling is disabled as the
         * users should know what they are doing.
         *
         * For v1, effective_cpus == cpus_allowed & user_xcpus() returns
         * cpus_allowed.
         *
         * For v2, is_cpu_exclusive() & is_sched_load_balance() are true only
         * for non-isolated partition root. At this point, the target
         * effective_cpus isn't computed yet. user_xcpus() is the best
         * approximation.
         *
         * TBD: May need to precompute the real effective_cpus here in case
         * incorrect scheduling of SCHED_DEADLINE tasks in a partition
         * becomes an issue.
         */
        ret = -EBUSY;
        if (is_cpu_exclusive(cur) && is_sched_load_balance(cur) &&
            !cpuset_cpumask_can_shrink(cur->effective_cpus, user_xcpus(trial)))
                goto out;

        /*
         * If either I or some sibling (!= me) is exclusive, we can't
         * overlap. exclusive_cpus cannot overlap with each other if set.
         */
        ret = -EINVAL;
        cpuset_for_each_child(c, css, par) {
                bool txset, cxset;        /* Are exclusive_cpus set? */

                if (c == cur)
                        continue;

                txset = !cpumask_empty(trial->exclusive_cpus);
                cxset = !cpumask_empty(c->exclusive_cpus);
                if (is_cpu_exclusive(trial) || is_cpu_exclusive(c) ||
                    (txset && cxset)) {
                        if (!cpusets_are_exclusive(trial, c))
                                goto out;
                } else if (txset || cxset) {
                        struct cpumask *xcpus, *acpus;

                        /*
                         * When just one of the exclusive_cpus's is set,
                         * cpus_allowed of the other cpuset, if set, cannot be
                         * a subset of it or none of those CPUs will be
                         * available if these exclusive CPUs are activated.
                         */
                        if (txset) {
                                xcpus = trial->exclusive_cpus;
                                acpus = c->cpus_allowed;
                        } else {
                                xcpus = c->exclusive_cpus;
                                acpus = trial->cpus_allowed;
                        }
                        if (!cpumask_empty(acpus) && cpumask_subset(acpus, xcpus))
                                goto out;
                }
                if ((is_mem_exclusive(trial) || is_mem_exclusive(c)) &&
                    nodes_intersects(trial->mems_allowed, c->mems_allowed))
                        goto out;
        }

        ret = 0;
out:
        rcu_read_unlock();
        return ret;
}

#ifdef CONFIG_SMP
/*
 * Helper routine for generate_sched_domains().
 * Do cpusets a, b have overlapping effective cpus_allowed masks?
 */
static int cpusets_overlap(struct cpuset *a, struct cpuset *b)
{
        return cpumask_intersects(a->effective_cpus, b->effective_cpus);
}

static void
update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c)
{
        if (dattr->relax_domain_level < c->relax_domain_level)
                dattr->relax_domain_level = c->relax_domain_level;
        return;
}

static void update_domain_attr_tree(struct sched_domain_attr *dattr,
                                    struct cpuset *root_cs)
{
        struct cpuset *cp;
        struct cgroup_subsys_state *pos_css;

        rcu_read_lock();
        cpuset_for_each_descendant_pre(cp, pos_css, root_cs) {
                /* skip the whole subtree if @cp doesn't have any CPU */
                if (cpumask_empty(cp->cpus_allowed)) {
                        pos_css = css_rightmost_descendant(pos_css);
                        continue;
                }

                if (is_sched_load_balance(cp))
                        update_domain_attr(dattr, cp);
        }
        rcu_read_unlock();
}

/* Must be called with cpuset_mutex held.  */
static inline int nr_cpusets(void)
{
        /* jump label reference count + the top-level cpuset */
        return static_key_count(&cpusets_enabled_key.key) + 1;
}

/*
 * generate_sched_domains()
 *
 * This function builds a partial partition of the systems CPUs
 * A 'partial partition' is a set of non-overlapping subsets whose
 * union is a subset of that set.
 * The output of this function needs to be passed to kernel/sched/core.c
 * partition_sched_domains() routine, which will rebuild the scheduler's
 * load balancing domains (sched domains) as specified by that partial
 * partition.
 *
 * See "What is sched_load_balance" in Documentation/admin-guide/cgroup-v1/cpusets.rst
 * for a background explanation of this.
 *
 * Does not return errors, on the theory that the callers of this
 * routine would rather not worry about failures to rebuild sched
 * domains when operating in the severe memory shortage situations
 * that could cause allocation failures below.
 *
 * Must be called with cpuset_mutex held.
 *
 * The three key local variables below are:
 *    cp - cpuset pointer, used (together with pos_css) to perform a
 *           top-down scan of all cpusets. For our purposes, rebuilding
 *           the schedulers sched domains, we can ignore !is_sched_load_
 *           balance cpusets.
 *  csa  - (for CpuSet Array) Array of pointers to all the cpusets
 *           that need to be load balanced, for convenient iterative
 *           access by the subsequent code that finds the best partition,
 *           i.e the set of domains (subsets) of CPUs such that the
 *           cpus_allowed of every cpuset marked is_sched_load_balance
 *           is a subset of one of these domains, while there are as
 *           many such domains as possible, each as small as possible.
 * doms  - Conversion of 'csa' to an array of cpumasks, for passing to
 *           the kernel/sched/core.c routine partition_sched_domains() in a
 *           convenient format, that can be easily compared to the prior
 *           value to determine what partition elements (sched domains)
 *           were changed (added or removed.)
 *
 * Finding the best partition (set of domains):
 *        The double nested loops below over i, j scan over the load
 *        balanced cpusets (using the array of cpuset pointers in csa[])
 *        looking for pairs of cpusets that have overlapping cpus_allowed
 *        and merging them using a union-find algorithm.
 *
 *        The union of the cpus_allowed masks from the set of all cpusets
 *        having the same root then form the one element of the partition
 *        (one sched domain) to be passed to partition_sched_domains().
 *
 */
static int generate_sched_domains(cpumask_var_t **domains,
                        struct sched_domain_attr **attributes)
{
        struct cpuset *cp;        /* top-down scan of cpusets */
        struct cpuset **csa;        /* array of all cpuset ptrs */
        int csn;                /* how many cpuset ptrs in csa so far */
        int i, j;                /* indices for partition finding loops */
        cpumask_var_t *doms;        /* resulting partition; i.e. sched domains */
        struct sched_domain_attr *dattr;  /* attributes for custom domains */
        int ndoms = 0;                /* number of sched domains in result */
        int nslot;                /* next empty doms[] struct cpumask slot */
        struct cgroup_subsys_state *pos_css;
        bool root_load_balance = is_sched_load_balance(&top_cpuset);
        bool cgrpv2 = cpuset_v2();
        int nslot_update;

        doms = NULL;
        dattr = NULL;
        csa = NULL;

        /* Special case for the 99% of systems with one, full, sched domain */
        if (root_load_balance && cpumask_empty(subpartitions_cpus)) {
single_root_domain:
                ndoms = 1;
                doms = alloc_sched_domains(ndoms);
                if (!doms)
                        goto done;

                dattr = kmalloc(sizeof(struct sched_domain_attr), GFP_KERNEL);
                if (dattr) {
                        *dattr = SD_ATTR_INIT;
                        update_domain_attr_tree(dattr, &top_cpuset);
                }
                cpumask_and(doms[0], top_cpuset.effective_cpus,
                            housekeeping_cpumask(HK_TYPE_DOMAIN));

                goto done;
        }

        csa = kmalloc_array(nr_cpusets(), sizeof(cp), GFP_KERNEL);
        if (!csa)
                goto done;
        csn = 0;

        rcu_read_lock();
        if (root_load_balance)
                csa[csn++] = &top_cpuset;
        cpuset_for_each_descendant_pre(cp, pos_css, &top_cpuset) {
                if (cp == &top_cpuset)
                        continue;

                if (cgrpv2)
                        goto v2;

                /*
                 * v1:
                 * Continue traversing beyond @cp iff @cp has some CPUs and
                 * isn't load balancing.  The former is obvious.  The
                 * latter: All child cpusets contain a subset of the
                 * parent's cpus, so just skip them, and then we call
                 * update_domain_attr_tree() to calc relax_domain_level of
                 * the corresponding sched domain.
                 */
                if (!cpumask_empty(cp->cpus_allowed) &&
                    !(is_sched_load_balance(cp) &&
                      cpumask_intersects(cp->cpus_allowed,
                                         housekeeping_cpumask(HK_TYPE_DOMAIN))))
                        continue;

                if (is_sched_load_balance(cp) &&
                    !cpumask_empty(cp->effective_cpus))
                        csa[csn++] = cp;

                /* skip @cp's subtree */
                pos_css = css_rightmost_descendant(pos_css);
                continue;

v2:
                /*
                 * Only valid partition roots that are not isolated and with
                 * non-empty effective_cpus will be saved into csn[].
                 */
                if ((cp->partition_root_state == PRS_ROOT) &&
                    !cpumask_empty(cp->effective_cpus))
                        csa[csn++] = cp;

                /*
                 * Skip @cp's subtree if not a partition root and has no
                 * exclusive CPUs to be granted to child cpusets.
                 */
                if (!is_partition_valid(cp) && cpumask_empty(cp->exclusive_cpus))
                        pos_css = css_rightmost_descendant(pos_css);
        }
        rcu_read_unlock();

        /*
         * If there are only isolated partitions underneath the cgroup root,
         * we can optimize out unneeded sched domains scanning.
         */
        if (root_load_balance && (csn == 1))
                goto single_root_domain;

        for (i = 0; i < csn; i++)
                uf_node_init(&csa[i]->node);

        /* Merge overlapping cpusets */
        for (i = 0; i < csn; i++) {
                for (j = i + 1; j < csn; j++) {
                        if (cpusets_overlap(csa[i], csa[j])) {
                                /*
                                 * Cgroup v2 shouldn't pass down overlapping
                                 * partition root cpusets.
                                 */
                                WARN_ON_ONCE(cgrpv2);
                                uf_union(&csa[i]->node, &csa[j]->node);
                        }
                }
        }

        /* Count the total number of domains */
        for (i = 0; i < csn; i++) {
                if (uf_find(&csa[i]->node) == &csa[i]->node)
                        ndoms++;
        }

        /*
         * Now we know how many domains to create.
         * Convert <csn, csa> to <ndoms, doms> and populate cpu masks.
         */
        doms = alloc_sched_domains(ndoms);
        if (!doms)
                goto done;

        /*
         * The rest of the code, including the scheduler, can deal with
         * dattr==NULL case. No need to abort if alloc fails.
         */
        dattr = kmalloc_array(ndoms, sizeof(struct sched_domain_attr),
                              GFP_KERNEL);

        /*
         * Cgroup v2 doesn't support domain attributes, just set all of them
         * to SD_ATTR_INIT. Also non-isolating partition root CPUs are a
         * subset of HK_TYPE_DOMAIN housekeeping CPUs.
         */
        if (cgrpv2) {
                for (i = 0; i < ndoms; i++) {
                        /*
                         * The top cpuset may contain some boot time isolated
                         * CPUs that need to be excluded from the sched domain.
                         */
                        if (csa[i] == &top_cpuset)
                                cpumask_and(doms[i], csa[i]->effective_cpus,
                                            housekeeping_cpumask(HK_TYPE_DOMAIN));
                        else
                                cpumask_copy(doms[i], csa[i]->effective_cpus);
                        if (dattr)
                                dattr[i] = SD_ATTR_INIT;
                }
                goto done;
        }

        for (nslot = 0, i = 0; i < csn; i++) {
                nslot_update = 0;
                for (j = i; j < csn; j++) {
                        if (uf_find(&csa[j]->node) == &csa[i]->node) {
                                struct cpumask *dp = doms[nslot];

                                if (i == j) {
                                        nslot_update = 1;
                                        cpumask_clear(dp);
                                        if (dattr)
                                                *(dattr + nslot) = SD_ATTR_INIT;
                                }
                                cpumask_or(dp, dp, csa[j]->effective_cpus);
                                cpumask_and(dp, dp, housekeeping_cpumask(HK_TYPE_DOMAIN));
                                if (dattr)
                                        update_domain_attr_tree(dattr + nslot, csa[j]);
                        }
                }
                if (nslot_update)
                        nslot++;
        }
        BUG_ON(nslot != ndoms);

done:
        kfree(csa);

        /*
         * Fallback to the default domain if kmalloc() failed.
         * See comments in partition_sched_domains().
         */
        if (doms == NULL)
                ndoms = 1;

        *domains    = doms;
        *attributes = dattr;
        return ndoms;
}

static void dl_update_tasks_root_domain(struct cpuset *cs)
{
        struct css_task_iter it;
        struct task_struct *task;

        if (cs->nr_deadline_tasks == 0)
                return;

        css_task_iter_start(&cs->css, 0, &it);

        while ((task = css_task_iter_next(&it)))
                dl_add_task_root_domain(task);

        css_task_iter_end(&it);
}

void dl_rebuild_rd_accounting(void)
{
        struct cpuset *cs = NULL;
        struct cgroup_subsys_state *pos_css;
        int cpu;
        u64 cookie = ++dl_cookie;

        lockdep_assert_held(&cpuset_mutex);
        lockdep_assert_cpus_held();
        lockdep_assert_held(&sched_domains_mutex);

        rcu_read_lock();

        for_each_possible_cpu(cpu) {
                if (dl_bw_visited(cpu, cookie))
                        continue;

                dl_clear_root_domain_cpu(cpu);
        }

        cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {

                if (cpumask_empty(cs->effective_cpus)) {
                        pos_css = css_rightmost_descendant(pos_css);
                        continue;
                }

                css_get(&cs->css);

                rcu_read_unlock();

                dl_update_tasks_root_domain(cs);

                rcu_read_lock();
                css_put(&cs->css);
        }
        rcu_read_unlock();
}

/*
 * Rebuild scheduler domains.
 *
 * If the flag 'sched_load_balance' of any cpuset with non-empty
 * 'cpus' changes, or if the 'cpus' allowed changes in any cpuset
 * which has that flag enabled, or if any cpuset with a non-empty
 * 'cpus' is removed, then call this routine to rebuild the
 * scheduler's dynamic sched domains.
 *
 * Call with cpuset_mutex held.  Takes cpus_read_lock().
 */
void rebuild_sched_domains_locked(void)
{
        struct cgroup_subsys_state *pos_css;
        struct sched_domain_attr *attr;
        cpumask_var_t *doms;
        struct cpuset *cs;
        int ndoms;

        lockdep_assert_cpus_held();
        lockdep_assert_held(&cpuset_mutex);
        force_sd_rebuild = false;

        /*
         * If we have raced with CPU hotplug, return early to avoid
         * passing doms with offlined cpu to partition_sched_domains().
         * Anyways, cpuset_handle_hotplug() will rebuild sched domains.
         *
         * With no CPUs in any subpartitions, top_cpuset's effective CPUs
         * should be the same as the active CPUs, so checking only top_cpuset
         * is enough to detect racing CPU offlines.
         */
        if (cpumask_empty(subpartitions_cpus) &&
            !cpumask_equal(top_cpuset.effective_cpus, cpu_active_mask))
                return;

        /*
         * With subpartition CPUs, however, the effective CPUs of a partition
         * root should be only a subset of the active CPUs.  Since a CPU in any
         * partition root could be offlined, all must be checked.
         */
        if (!cpumask_empty(subpartitions_cpus)) {
                rcu_read_lock();
                cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
                        if (!is_partition_valid(cs)) {
                                pos_css = css_rightmost_descendant(pos_css);
                                continue;
                        }
                        if (!cpumask_subset(cs->effective_cpus,
                                            cpu_active_mask)) {
                                rcu_read_unlock();
                                return;
                        }
                }
                rcu_read_unlock();
        }

        /* Generate domain masks and attrs */
        ndoms = generate_sched_domains(&doms, &attr);

        /* Have scheduler rebuild the domains */
        partition_sched_domains(ndoms, doms, attr);
}
#else /* !CONFIG_SMP */
void rebuild_sched_domains_locked(void)
{
}
#endif /* CONFIG_SMP */

static void rebuild_sched_domains_cpuslocked(void)
{
        mutex_lock(&cpuset_mutex);
        rebuild_sched_domains_locked();
        mutex_unlock(&cpuset_mutex);
}

void rebuild_sched_domains(void)
{
        cpus_read_lock();
        rebuild_sched_domains_cpuslocked();
        cpus_read_unlock();
}

void cpuset_reset_sched_domains(void)
{
        mutex_lock(&cpuset_mutex);
        partition_sched_domains(1, NULL, NULL);
        mutex_unlock(&cpuset_mutex);
}

/**
 * cpuset_update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
 * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
 * @new_cpus: the temp variable for the new effective_cpus mask
 *
 * Iterate through each task of @cs updating its cpus_allowed to the
 * effective cpuset's.  As this function is called with cpuset_mutex held,
 * cpuset membership stays stable.
 *
 * For top_cpuset, task_cpu_possible_mask() is used instead of effective_cpus
 * to make sure all offline CPUs are also included as hotplug code won't
 * update cpumasks for tasks in top_cpuset.
 *
 * As task_cpu_possible_mask() can be task dependent in arm64, we have to
 * do cpu masking per task instead of doing it once for all.
 */
void cpuset_update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus)
{
        struct css_task_iter it;
        struct task_struct *task;
        bool top_cs = cs == &top_cpuset;

        css_task_iter_start(&cs->css, 0, &it);
        while ((task = css_task_iter_next(&it))) {
                const struct cpumask *possible_mask = task_cpu_possible_mask(task);

                if (top_cs) {
                        /*
                         * Percpu kthreads in top_cpuset are ignored
                         */
                        if (kthread_is_per_cpu(task))
                                continue;
                        cpumask_andnot(new_cpus, possible_mask, subpartitions_cpus);
                } else {
                        cpumask_and(new_cpus, possible_mask, cs->effective_cpus);
                }
                set_cpus_allowed_ptr(task, new_cpus);
        }
        css_task_iter_end(&it);
}

/**
 * compute_effective_cpumask - Compute the effective cpumask of the cpuset
 * @new_cpus: the temp variable for the new effective_cpus mask
 * @cs: the cpuset the need to recompute the new effective_cpus mask
 * @parent: the parent cpuset
 *
 * The result is valid only if the given cpuset isn't a partition root.
 */
static void compute_effective_cpumask(struct cpumask *new_cpus,
                                      struct cpuset *cs, struct cpuset *parent)
{
        cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus);
}

/*
 * Commands for update_parent_effective_cpumask
 */
enum partition_cmd {
        partcmd_enable,                /* Enable partition root          */
        partcmd_enablei,        /* Enable isolated partition root */
        partcmd_disable,        /* Disable partition root          */
        partcmd_update,                /* Update parent's effective_cpus */
        partcmd_invalidate,        /* Make partition invalid          */
};

static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs,
                                    struct tmpmasks *tmp);

/*
 * Update partition exclusive flag
 *
 * Return: 0 if successful, an error code otherwise
 */
static int update_partition_exclusive_flag(struct cpuset *cs, int new_prs)
{
        bool exclusive = (new_prs > PRS_MEMBER);

        if (exclusive && !is_cpu_exclusive(cs)) {
                if (cpuset_update_flag(CS_CPU_EXCLUSIVE, cs, 1))
                        return PERR_NOTEXCL;
        } else if (!exclusive && is_cpu_exclusive(cs)) {
                /* Turning off CS_CPU_EXCLUSIVE will not return error */
                cpuset_update_flag(CS_CPU_EXCLUSIVE, cs, 0);
        }
        return 0;
}

/*
 * Update partition load balance flag and/or rebuild sched domain
 *
 * Changing load balance flag will automatically call
 * rebuild_sched_domains_locked().
 * This function is for cgroup v2 only.
 */
static void update_partition_sd_lb(struct cpuset *cs, int old_prs)
{
        int new_prs = cs->partition_root_state;
        bool rebuild_domains = (new_prs > 0) || (old_prs > 0);
        bool new_lb;

        /*
         * If cs is not a valid partition root, the load balance state
         * will follow its parent.
         */
        if (new_prs > 0) {
                new_lb = (new_prs != PRS_ISOLATED);
        } else {
                new_lb = is_sched_load_balance(parent_cs(cs));
        }
        if (new_lb != !!is_sched_load_balance(cs)) {
                rebuild_domains = true;
                if (new_lb)
                        set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
                else
                        clear_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
        }

        if (rebuild_domains)
                cpuset_force_rebuild();
}

/*
 * tasks_nocpu_error - Return true if tasks will have no effective_cpus
 */
static bool tasks_nocpu_error(struct cpuset *parent, struct cpuset *cs,
                              struct cpumask *xcpus)
{
        /*
         * A populated partition (cs or parent) can't have empty effective_cpus
         */
        return (cpumask_subset(parent->effective_cpus, xcpus) &&
                partition_is_populated(parent, cs)) ||
               (!cpumask_intersects(xcpus, cpu_active_mask) &&
                partition_is_populated(cs, NULL));
}

static void reset_partition_data(struct cpuset *cs)
{
        struct cpuset *parent = parent_cs(cs);

        if (!cpuset_v2())
                return;

        lockdep_assert_held(&callback_lock);

        cs->nr_subparts = 0;
        if (cpumask_empty(cs->exclusive_cpus)) {
                cpumask_clear(cs->effective_xcpus);
                if (is_cpu_exclusive(cs))
                        clear_bit(CS_CPU_EXCLUSIVE, &cs->flags);
        }
        if (!cpumask_and(cs->effective_cpus, parent->effective_cpus, cs->cpus_allowed))
                cpumask_copy(cs->effective_cpus, parent->effective_cpus);
}

/*
 * isolated_cpus_update - Update the isolated_cpus mask
 * @old_prs: old partition_root_state
 * @new_prs: new partition_root_state
 * @xcpus: exclusive CPUs with state change
 */
static void isolated_cpus_update(int old_prs, int new_prs, struct cpumask *xcpus)
{
        WARN_ON_ONCE(old_prs == new_prs);
        if (new_prs == PRS_ISOLATED)
                cpumask_or(isolated_cpus, isolated_cpus, xcpus);
        else
                cpumask_andnot(isolated_cpus, isolated_cpus, xcpus);
}

/*
 * partition_xcpus_add - Add new exclusive CPUs to partition
 * @new_prs: new partition_root_state
 * @parent: parent cpuset
 * @xcpus: exclusive CPUs to be added
 * Return: true if isolated_cpus modified, false otherwise
 *
 * Remote partition if parent == NULL
 */
static bool partition_xcpus_add(int new_prs, struct cpuset *parent,
                                struct cpumask *xcpus)
{
        bool isolcpus_updated;

        WARN_ON_ONCE(new_prs < 0);
        lockdep_assert_held(&callback_lock);
        if (!parent)
                parent = &top_cpuset;


        if (parent == &top_cpuset)
                cpumask_or(subpartitions_cpus, subpartitions_cpus, xcpus);

        isolcpus_updated = (new_prs != parent->partition_root_state);
        if (isolcpus_updated)
                isolated_cpus_update(parent->partition_root_state, new_prs,
                                     xcpus);

        cpumask_andnot(parent->effective_cpus, parent->effective_cpus, xcpus);
        return isolcpus_updated;
}

/*
 * partition_xcpus_del - Remove exclusive CPUs from partition
 * @old_prs: old partition_root_state
 * @parent: parent cpuset
 * @xcpus: exclusive CPUs to be removed
 * Return: true if isolated_cpus modified, false otherwise
 *
 * Remote partition if parent == NULL
 */
static bool partition_xcpus_del(int old_prs, struct cpuset *parent,
                                struct cpumask *xcpus)
{
        bool isolcpus_updated;

        WARN_ON_ONCE(old_prs < 0);
        lockdep_assert_held(&callback_lock);
        if (!parent)
                parent = &top_cpuset;

        if (parent == &top_cpuset)
                cpumask_andnot(subpartitions_cpus, subpartitions_cpus, xcpus);

        isolcpus_updated = (old_prs != parent->partition_root_state);
        if (isolcpus_updated)
                isolated_cpus_update(old_prs, parent->partition_root_state,
                                     xcpus);

        cpumask_and(xcpus, xcpus, cpu_active_mask);
        cpumask_or(parent->effective_cpus, parent->effective_cpus, xcpus);
        return isolcpus_updated;
}

static void update_unbound_workqueue_cpumask(bool isolcpus_updated)
{
        int ret;

        lockdep_assert_cpus_held();

        if (!isolcpus_updated)
                return;

        ret = workqueue_unbound_exclude_cpumask(isolated_cpus);
        WARN_ON_ONCE(ret < 0);
}

/**
 * cpuset_cpu_is_isolated - Check if the given CPU is isolated
 * @cpu: the CPU number to be checked
 * Return: true if CPU is used in an isolated partition, false otherwise
 */
bool cpuset_cpu_is_isolated(int cpu)
{
        return cpumask_test_cpu(cpu, isolated_cpus);
}
EXPORT_SYMBOL_GPL(cpuset_cpu_is_isolated);

/*
 * compute_effective_exclusive_cpumask - compute effective exclusive CPUs
 * @cs: cpuset
 * @xcpus: effective exclusive CPUs value to be set
 * @real_cs: the real cpuset (can be NULL)
 * Return: 0 if there is no sibling conflict, > 0 otherwise
 *
 * If exclusive_cpus isn't explicitly set or a real_cs is provided, we have to
 * scan the sibling cpusets and exclude their exclusive_cpus or effective_xcpus
 * as well. The provision of real_cs means that a cpumask is being changed and
 * the given cs is a trial one.
 */
static int compute_effective_exclusive_cpumask(struct cpuset *cs,
                                               struct cpumask *xcpus,
                                               struct cpuset *real_cs)
{
        struct cgroup_subsys_state *css;
        struct cpuset *parent = parent_cs(cs);
        struct cpuset *sibling;
        int retval = 0;

        if (!xcpus)
                xcpus = cs->effective_xcpus;

        cpumask_and(xcpus, user_xcpus(cs), parent->effective_xcpus);

        if (!real_cs) {
                if (!cpumask_empty(cs->exclusive_cpus))
                        return 0;
        } else {
                cs = real_cs;
        }

        /*
         * Exclude exclusive CPUs from siblings
         */
        rcu_read_lock();
        cpuset_for_each_child(sibling, css, parent) {
                if (sibling == cs)
                        continue;

                if (!cpumask_empty(sibling->exclusive_cpus) &&
                    cpumask_intersects(xcpus, sibling->exclusive_cpus)) {
                        cpumask_andnot(xcpus, xcpus, sibling->exclusive_cpus);
                        retval++;
                        continue;
                }
                if (!cpumask_empty(sibling->effective_xcpus) &&
                    cpumask_intersects(xcpus, sibling->effective_xcpus)) {
                        cpumask_andnot(xcpus, xcpus, sibling->effective_xcpus);
                        retval++;
                }
        }
        rcu_read_unlock();
        return retval;
}

static inline bool is_remote_partition(struct cpuset *cs)
{
        return !list_empty(&cs->remote_sibling);
}

static inline bool is_local_partition(struct cpuset *cs)
{
        return is_partition_valid(cs) && !is_remote_partition(cs);
}

/*
 * remote_partition_enable - Enable current cpuset as a remote partition root
 * @cs: the cpuset to update
 * @new_prs: new partition_root_state
 * @tmp: temporary masks
 * Return: 0 if successful, errcode if error
 *
 * Enable the current cpuset to become a remote partition root taking CPUs
 * directly from the top cpuset. cpuset_mutex must be held by the caller.
 */
static int remote_partition_enable(struct cpuset *cs, int new_prs,
                                   struct tmpmasks *tmp)
{
        bool isolcpus_updated;

        /*
         * The user must have sysadmin privilege.
         */
        if (!capable(CAP_SYS_ADMIN))
                return PERR_ACCESS;

        /*
         * The requested exclusive_cpus must not be allocated to other
         * partitions and it can't use up all the root's effective_cpus.
         *
         * Note that if there is any local partition root above it or
         * remote partition root underneath it, its exclusive_cpus must
         * have overlapped with subpartitions_cpus.
         */
        compute_effective_exclusive_cpumask(cs, tmp->new_cpus, NULL);
        if (cpumask_empty(tmp->new_cpus) ||
            cpumask_intersects(tmp->new_cpus, subpartitions_cpus) ||
            cpumask_subset(top_cpuset.effective_cpus, tmp->new_cpus))
                return PERR_INVCPUS;

        spin_lock_irq(&callback_lock);
        isolcpus_updated = partition_xcpus_add(new_prs, NULL, tmp->new_cpus);
        list_add(&cs->remote_sibling, &remote_children);
        cpumask_copy(cs->effective_xcpus, tmp->new_cpus);
        spin_unlock_irq(&callback_lock);
        update_unbound_workqueue_cpumask(isolcpus_updated);
        cpuset_force_rebuild();
        cs->prs_err = 0;

        /*
         * Propagate changes in top_cpuset's effective_cpus down the hierarchy.
         */
        cpuset_update_tasks_cpumask(&top_cpuset, tmp->new_cpus);
        update_sibling_cpumasks(&top_cpuset, NULL, tmp);
        return 0;
}

/*
 * remote_partition_disable - Remove current cpuset from remote partition list
 * @cs: the cpuset to update
 * @tmp: temporary masks
 *
 * The effective_cpus is also updated.
 *
 * cpuset_mutex must be held by the caller.
 */
static void remote_partition_disable(struct cpuset *cs, struct tmpmasks *tmp)
{
        bool isolcpus_updated;

        WARN_ON_ONCE(!is_remote_partition(cs));
        WARN_ON_ONCE(!cpumask_subset(cs->effective_xcpus, subpartitions_cpus));

        spin_lock_irq(&callback_lock);
        list_del_init(&cs->remote_sibling);
        isolcpus_updated = partition_xcpus_del(cs->partition_root_state,
                                               NULL, cs->effective_xcpus);
        if (cs->prs_err)
                cs->partition_root_state = -cs->partition_root_state;
        else
                cs->partition_root_state = PRS_MEMBER;

        /* effective_xcpus may need to be changed */
        compute_effective_exclusive_cpumask(cs, NULL, NULL);
        reset_partition_data(cs);
        spin_unlock_irq(&callback_lock);
        update_unbound_workqueue_cpumask(isolcpus_updated);
        cpuset_force_rebuild();

        /*
         * Propagate changes in top_cpuset's effective_cpus down the hierarchy.
         */
        cpuset_update_tasks_cpumask(&top_cpuset, tmp->new_cpus);
        update_sibling_cpumasks(&top_cpuset, NULL, tmp);
}

/*
 * remote_cpus_update - cpus_exclusive change of remote partition
 * @cs: the cpuset to be updated
 * @xcpus: the new exclusive_cpus mask, if non-NULL
 * @excpus: the new effective_xcpus mask
 * @tmp: temporary masks
 *
 * top_cpuset and subpartitions_cpus will be updated or partition can be
 * invalidated.
 */
static void remote_cpus_update(struct cpuset *cs, struct cpumask *xcpus,
                               struct cpumask *excpus, struct tmpmasks *tmp)
{
        bool adding, deleting;
        int prs = cs->partition_root_state;
        int isolcpus_updated = 0;

        if (WARN_ON_ONCE(!is_remote_partition(cs)))
                return;

        WARN_ON_ONCE(!cpumask_subset(cs->effective_xcpus, subpartitions_cpus));

        if (cpumask_empty(excpus)) {
                cs->prs_err = PERR_CPUSEMPTY;
                goto invalidate;
        }

        adding   = cpumask_andnot(tmp->addmask, excpus, cs->effective_xcpus);
        deleting = cpumask_andnot(tmp->delmask, cs->effective_xcpus, excpus);

        /*
         * Additions of remote CPUs is only allowed if those CPUs are
         * not allocated to other partitions and there are effective_cpus
         * left in the top cpuset.
         */
        if (adding) {
                if (!capable(CAP_SYS_ADMIN))
                        cs->prs_err = PERR_ACCESS;
                else if (cpumask_intersects(tmp->addmask, subpartitions_cpus) ||
                         cpumask_subset(top_cpuset.effective_cpus, tmp->addmask))
                        cs->prs_err = PERR_NOCPUS;
                if (cs->prs_err)
                        goto invalidate;
        }

        spin_lock_irq(&callback_lock);
        if (adding)
                isolcpus_updated += partition_xcpus_add(prs, NULL, tmp->addmask);
        if (deleting)
                isolcpus_updated += partition_xcpus_del(prs, NULL, tmp->delmask);
        /*
         * Need to update effective_xcpus and exclusive_cpus now as
         * update_sibling_cpumasks() below may iterate back to the same cs.
         */
        cpumask_copy(cs->effective_xcpus, excpus);
        if (xcpus)
                cpumask_copy(cs->exclusive_cpus, xcpus);
        spin_unlock_irq(&callback_lock);
        update_unbound_workqueue_cpumask(isolcpus_updated);
        if (adding || deleting)
                cpuset_force_rebuild();

        /*
         * Propagate changes in top_cpuset's effective_cpus down the hierarchy.
         */
        cpuset_update_tasks_cpumask(&top_cpuset, tmp->new_cpus);
        update_sibling_cpumasks(&top_cpuset, NULL, tmp);
        return;

invalidate:
        remote_partition_disable(cs, tmp);
}

/*
 * prstate_housekeeping_conflict - check for partition & housekeeping conflicts
 * @prstate: partition root state to be checked
 * @new_cpus: cpu mask
 * Return: true if there is conflict, false otherwise
 *
 * CPUs outside of boot_hk_cpus, if defined, can only be used in an
 * isolated partition.
 */
static bool prstate_housekeeping_conflict(int prstate, struct cpumask *new_cpus)
{
        if (!have_boot_isolcpus)
                return false;

        if ((prstate != PRS_ISOLATED) && !cpumask_subset(new_cpus, boot_hk_cpus))
                return true;

        return false;
}

/**
 * update_parent_effective_cpumask - update effective_cpus mask of parent cpuset
 * @cs:      The cpuset that requests change in partition root state
 * @cmd:     Partition root state change command
 * @newmask: Optional new cpumask for partcmd_update
 * @tmp:     Temporary addmask and delmask
 * Return:   0 or a partition root state error code
 *
 * For partcmd_enable*, the cpuset is being transformed from a non-partition
 * root to a partition root. The effective_xcpus (cpus_allowed if
 * effective_xcpus not set) mask of the given cpuset will be taken away from
 * parent's effective_cpus. The function will return 0 if all the CPUs listed
 * in effective_xcpus can be granted or an error code will be returned.
 *
 * For partcmd_disable, the cpuset is being transformed from a partition
 * root back to a non-partition root. Any CPUs in effective_xcpus will be
 * given back to parent's effective_cpus. 0 will always be returned.
 *
 * For partcmd_update, if the optional newmask is specified, the cpu list is
 * to be changed from effective_xcpus to newmask. Otherwise, effective_xcpus is
 * assumed to remain the same. The cpuset should either be a valid or invalid
 * partition root. The partition root state may change from valid to invalid
 * or vice versa. An error code will be returned if transitioning from
 * invalid to valid violates the exclusivity rule.
 *
 * For partcmd_invalidate, the current partition will be made invalid.
 *
 * The partcmd_enable* and partcmd_disable commands are used by
 * update_prstate(). An error code may be returned and the caller will check
 * for error.
 *
 * The partcmd_update command is used by update_cpumasks_hier() with newmask
 * NULL and update_cpumask() with newmask set. The partcmd_invalidate is used
 * by update_cpumask() with NULL newmask. In both cases, the callers won't
 * check for error and so partition_root_state and prs_err will be updated
 * directly.
 */
static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
                                           struct cpumask *newmask,
                                           struct tmpmasks *tmp)
{
        struct cpuset *parent = parent_cs(cs);
        int adding;        /* Adding cpus to parent's effective_cpus        */
        int deleting;        /* Deleting cpus from parent's effective_cpus        */
        int old_prs, new_prs;
        int part_error = PERR_NONE;        /* Partition error? */
        int subparts_delta = 0;
        int isolcpus_updated = 0;
        struct cpumask *xcpus = user_xcpus(cs);
        bool nocpu;

        lockdep_assert_held(&cpuset_mutex);
        WARN_ON_ONCE(is_remote_partition(cs));

        /*
         * new_prs will only be changed for the partcmd_update and
         * partcmd_invalidate commands.
         */
        adding = deleting = false;
        old_prs = new_prs = cs->partition_root_state;

        if (cmd == partcmd_invalidate) {
                if (is_prs_invalid(old_prs))
                        return 0;

                /*
                 * Make the current partition invalid.
                 */
                if (is_partition_valid(parent))
                        adding = cpumask_and(tmp->addmask,
                                             xcpus, parent->effective_xcpus);
                if (old_prs > 0) {
                        new_prs = -old_prs;
                        subparts_delta--;
                }
                goto write_error;
        }

        /*
         * The parent must be a partition root.
         * The new cpumask, if present, or the current cpus_allowed must
         * not be empty.
         */
        if (!is_partition_valid(parent)) {
                return is_partition_invalid(parent)
                       ? PERR_INVPARENT : PERR_NOTPART;
        }
        if (!newmask && xcpus_empty(cs))
                return PERR_CPUSEMPTY;

        nocpu = tasks_nocpu_error(parent, cs, xcpus);

        if ((cmd == partcmd_enable) || (cmd == partcmd_enablei)) {
                /*
                 * Need to call compute_effective_exclusive_cpumask() in case
                 * exclusive_cpus not set. Sibling conflict should only happen
                 * if exclusive_cpus isn't set.
                 */
                xcpus = tmp->new_cpus;
                if (compute_effective_exclusive_cpumask(cs, xcpus, NULL))
                        WARN_ON_ONCE(!cpumask_empty(cs->exclusive_cpus));

                /*
                 * Enabling partition root is not allowed if its
                 * effective_xcpus is empty.
                 */
                if (cpumask_empty(xcpus))
                        return PERR_INVCPUS;

                if (prstate_housekeeping_conflict(new_prs, xcpus))
                        return PERR_HKEEPING;

                /*
                 * A parent can be left with no CPU as long as there is no
                 * task directly associated with the parent partition.
                 */
                if (nocpu)
                        return PERR_NOCPUS;

                deleting = cpumask_and(tmp->delmask, xcpus, parent->effective_xcpus);
                if (deleting)
                        subparts_delta++;
                new_prs = (cmd == partcmd_enable) ? PRS_ROOT : PRS_ISOLATED;
        } else if (cmd == partcmd_disable) {
                /*
                 * May need to add cpus back to parent's effective_cpus
                 * (and maybe removed from subpartitions_cpus/isolated_cpus)
                 * for valid partition root. xcpus may contain CPUs that
                 * shouldn't be removed from the two global cpumasks.
                 */
                if (is_partition_valid(cs)) {
                        cpumask_copy(tmp->addmask, cs->effective_xcpus);
                        adding = true;
                        subparts_delta--;
                }
                new_prs = PRS_MEMBER;
        } else if (newmask) {
                /*
                 * Empty cpumask is not allowed
                 */
                if (cpumask_empty(newmask)) {
                        part_error = PERR_CPUSEMPTY;
                        goto write_error;
                }

                /* Check newmask again, whether cpus are available for parent/cs */
                nocpu |= tasks_nocpu_error(parent, cs, newmask);

                /*
                 * partcmd_update with newmask:
                 *
                 * Compute add/delete mask to/from effective_cpus
                 *
                 * For valid partition:
                 *   addmask = exclusive_cpus & ~newmask
                 *                              & parent->effective_xcpus
                 *   delmask = newmask & ~exclusive_cpus
                 *                       & parent->effective_xcpus
                 *
                 * For invalid partition:
                 *   delmask = newmask & parent->effective_xcpus
                 */
                if (is_prs_invalid(old_prs)) {
                        adding = false;
                        deleting = cpumask_and(tmp->delmask,
                                        newmask, parent->effective_xcpus);
                } else {
                        cpumask_andnot(tmp->addmask, xcpus, newmask);
                        adding = cpumask_and(tmp->addmask, tmp->addmask,
                                             parent->effective_xcpus);

                        cpumask_andnot(tmp->delmask, newmask, xcpus);
                        deleting = cpumask_and(tmp->delmask, tmp->delmask,
                                               parent->effective_xcpus);
                }
                /*
                 * Make partition invalid if parent's effective_cpus could
                 * become empty and there are tasks in the parent.
                 */
                if (nocpu && (!adding ||
                    !cpumask_intersects(tmp->addmask, cpu_active_mask))) {
                        part_error = PERR_NOCPUS;
                        deleting = false;
                        adding = cpumask_and(tmp->addmask,
                                             xcpus, parent->effective_xcpus);
                }
        } else {
                /*
                 * partcmd_update w/o newmask
                 *
                 * delmask = effective_xcpus & parent->effective_cpus
                 *
                 * This can be called from:
                 * 1) update_cpumasks_hier()
                 * 2) cpuset_hotplug_update_tasks()
                 *
                 * Check to see if it can be transitioned from valid to
                 * invalid partition or vice versa.
                 *
                 * A partition error happens when parent has tasks and all
                 * its effective CPUs will have to be distributed out.
                 */
                WARN_ON_ONCE(!is_partition_valid(parent));
                if (nocpu) {
                        part_error = PERR_NOCPUS;
                        if (is_partition_valid(cs))
                                adding = cpumask_and(tmp->addmask,
                                                xcpus, parent->effective_xcpus);
                } else if (is_partition_invalid(cs) &&
                           cpumask_subset(xcpus, parent->effective_xcpus)) {
                        struct cgroup_subsys_state *css;
                        struct cpuset *child;
                        bool exclusive = true;

                        /*
                         * Convert invalid partition to valid has to
                         * pass the cpu exclusivity test.
                         */
                        rcu_read_lock();
                        cpuset_for_each_child(child, css, parent) {
                                if (child == cs)
                                        continue;
                                if (!cpusets_are_exclusive(cs, child)) {
                                        exclusive = false;
                                        break;
                                }
                        }
                        rcu_read_unlock();
                        if (exclusive)
                                deleting = cpumask_and(tmp->delmask,
                                                xcpus, parent->effective_cpus);
                        else
                                part_error = PERR_NOTEXCL;
                }
        }

write_error:
        if (part_error)
                WRITE_ONCE(cs->prs_err, part_error);

        if (cmd == partcmd_update) {
                /*
                 * Check for possible transition between valid and invalid
                 * partition root.
                 */
                switch (cs->partition_root_state) {
                case PRS_ROOT:
                case PRS_ISOLATED:
                        if (part_error) {
                                new_prs = -old_prs;
                                subparts_delta--;
                        }
                        break;
                case PRS_INVALID_ROOT:
                case PRS_INVALID_ISOLATED:
                        if (!part_error) {
                                new_prs = -old_prs;
                                subparts_delta++;
                        }
                        break;
                }
        }

        if (!adding && !deleting && (new_prs == old_prs))
                return 0;

        /*
         * Transitioning between invalid to valid or vice versa may require
         * changing CS_CPU_EXCLUSIVE. In the case of partcmd_update,
         * validate_change() has already been successfully called and
         * CPU lists in cs haven't been updated yet. So defer it to later.
         */
        if ((old_prs != new_prs) && (cmd != partcmd_update))  {
                int err = update_partition_exclusive_flag(cs, new_prs);

                if (err)
                        return err;
        }

        /*
         * Change the parent's effective_cpus & effective_xcpus (top cpuset
         * only).
         *
         * Newly added CPUs will be removed from effective_cpus and
         * newly deleted ones will be added back to effective_cpus.
         */
        spin_lock_irq(&callback_lock);
        if (old_prs != new_prs) {
                cs->partition_root_state = new_prs;
                if (new_prs <= 0)
                        cs->nr_subparts = 0;
        }
        /*
         * Adding to parent's effective_cpus means deletion CPUs from cs
         * and vice versa.
         */
        if (adding)
                isolcpus_updated += partition_xcpus_del(old_prs, parent,
                                                        tmp->addmask);
        if (deleting)
                isolcpus_updated += partition_xcpus_add(new_prs, parent,
                                                        tmp->delmask);

        if (is_partition_valid(parent)) {
                parent->nr_subparts += subparts_delta;
                WARN_ON_ONCE(parent->nr_subparts < 0);
        }
        spin_unlock_irq(&callback_lock);
        update_unbound_workqueue_cpumask(isolcpus_updated);

        if ((old_prs != new_prs) && (cmd == partcmd_update))
                update_partition_exclusive_flag(cs, new_prs);

        if (adding || deleting) {
                cpuset_update_tasks_cpumask(parent, tmp->addmask);
                update_sibling_cpumasks(parent, cs, tmp);
        }

        /*
         * For partcmd_update without newmask, it is being called from
         * cpuset_handle_hotplug(). Update the load balance flag and
         * scheduling domain accordingly.
         */
        if ((cmd == partcmd_update) && !newmask)
                update_partition_sd_lb(cs, old_prs);

        notify_partition_change(cs, old_prs);
        return 0;
}

/**
 * compute_partition_effective_cpumask - compute effective_cpus for partition
 * @cs: partition root cpuset
 * @new_ecpus: previously computed effective_cpus to be updated
 *
 * Compute the effective_cpus of a partition root by scanning effective_xcpus
 * of child partition roots and excluding their effective_xcpus.
 *
 * This has the side effect of invalidating valid child partition roots,
 * if necessary. Since it is called from either cpuset_hotplug_update_tasks()
 * or update_cpumasks_hier() where parent and children are modified
 * successively, we don't need to call update_parent_effective_cpumask()
 * and the child's effective_cpus will be updated in later iterations.
 *
 * Note that rcu_read_lock() is assumed to be held.
 */
static void compute_partition_effective_cpumask(struct cpuset *cs,
                                                struct cpumask *new_ecpus)
{
        struct cgroup_subsys_state *css;
        struct cpuset *child;
        bool populated = partition_is_populated(cs, NULL);

        /*
         * Check child partition roots to see if they should be
         * invalidated when
         *  1) child effective_xcpus not a subset of new
         *     excluisve_cpus
         *  2) All the effective_cpus will be used up and cp
         *     has tasks
         */
        compute_effective_exclusive_cpumask(cs, new_ecpus, NULL);
        cpumask_and(new_ecpus, new_ecpus, cpu_active_mask);

        rcu_read_lock();
        cpuset_for_each_child(child, css, cs) {
                if (!is_partition_valid(child))
                        continue;

                /*
                 * There shouldn't be a remote partition underneath another
                 * partition root.
                 */
                WARN_ON_ONCE(is_remote_partition(child));
                child->prs_err = 0;
                if (!cpumask_subset(child->effective_xcpus,
                                    cs->effective_xcpus))
                        child->prs_err = PERR_INVCPUS;
                else if (populated &&
                         cpumask_subset(new_ecpus, child->effective_xcpus))
                        child->prs_err = PERR_NOCPUS;

                if (child->prs_err) {
                        int old_prs = child->partition_root_state;

                        /*
                         * Invalidate child partition
                         */
                        spin_lock_irq(&callback_lock);
                        make_partition_invalid(child);
                        cs->nr_subparts--;
                        child->nr_subparts = 0;
                        spin_unlock_irq(&callback_lock);
                        notify_partition_change(child, old_prs);
                        continue;
                }
                cpumask_andnot(new_ecpus, new_ecpus,
                               child->effective_xcpus);
        }
        rcu_read_unlock();
}

/*
 * update_cpumasks_hier - Update effective cpumasks and tasks in the subtree
 * @cs:  the cpuset to consider
 * @tmp: temp variables for calculating effective_cpus & partition setup
 * @force: don't skip any descendant cpusets if set
 *
 * When configured cpumask is changed, the effective cpumasks of this cpuset
 * and all its descendants need to be updated.
 *
 * On legacy hierarchy, effective_cpus will be the same with cpu_allowed.
 *
 * Called with cpuset_mutex held
 */
static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp,
                                 bool force)
{
        struct cpuset *cp;
        struct cgroup_subsys_state *pos_css;
        bool need_rebuild_sched_domains = false;
        int old_prs, new_prs;

        rcu_read_lock();
        cpuset_for_each_descendant_pre(cp, pos_css, cs) {
                struct cpuset *parent = parent_cs(cp);
                bool remote = is_remote_partition(cp);
                bool update_parent = false;

                old_prs = new_prs = cp->partition_root_state;

                /*
                 * For child remote partition root (!= cs), we need to call
                 * remote_cpus_update() if effective_xcpus will be changed.
                 * Otherwise, we can skip the whole subtree.
                 *
                 * remote_cpus_update() will reuse tmp->new_cpus only after
                 * its value is being processed.
                 */
                if (remote && (cp != cs)) {
                        compute_effective_exclusive_cpumask(cp, tmp->new_cpus, NULL);
                        if (cpumask_equal(cp->effective_xcpus, tmp->new_cpus)) {
                                pos_css = css_rightmost_descendant(pos_css);
                                continue;
                        }
                        rcu_read_unlock();
                        remote_cpus_update(cp, NULL, tmp->new_cpus, tmp);
                        rcu_read_lock();

                        /* Remote partition may be invalidated */
                        new_prs = cp->partition_root_state;
                        remote = (new_prs == old_prs);
                }

                if (remote || (is_partition_valid(parent) && is_partition_valid(cp)))
                        compute_partition_effective_cpumask(cp, tmp->new_cpus);
                else
                        compute_effective_cpumask(tmp->new_cpus, cp, parent);

                if (remote)
                        goto get_css;        /* Ready to update cpuset data */

                /*
                 * A partition with no effective_cpus is allowed as long as
                 * there is no task associated with it. Call
                 * update_parent_effective_cpumask() to check it.
                 */
                if (is_partition_valid(cp) && cpumask_empty(tmp->new_cpus)) {
                        update_parent = true;
                        goto update_parent_effective;
                }

                /*
                 * If it becomes empty, inherit the effective mask of the
                 * parent, which is guaranteed to have some CPUs unless
                 * it is a partition root that has explicitly distributed
                 * out all its CPUs.
                 */
                if (is_in_v2_mode() && !remote && cpumask_empty(tmp->new_cpus))
                        cpumask_copy(tmp->new_cpus, parent->effective_cpus);

                /*
                 * Skip the whole subtree if
                 * 1) the cpumask remains the same,
                 * 2) has no partition root state,
                 * 3) force flag not set, and
                 * 4) for v2 load balance state same as its parent.
                 */
                if (!cp->partition_root_state && !force &&
                    cpumask_equal(tmp->new_cpus, cp->effective_cpus) &&
                    (!cpuset_v2() ||
                    (is_sched_load_balance(parent) == is_sched_load_balance(cp)))) {
                        pos_css = css_rightmost_descendant(pos_css);
                        continue;
                }

update_parent_effective:
                /*
                 * update_parent_effective_cpumask() should have been called
                 * for cs already in update_cpumask(). We should also call
                 * cpuset_update_tasks_cpumask() again for tasks in the parent
                 * cpuset if the parent's effective_cpus changes.
                 */
                if ((cp != cs) && old_prs) {
                        switch (parent->partition_root_state) {
                        case PRS_ROOT:
                        case PRS_ISOLATED:
                                update_parent = true;
                                break;

                        default:
                                /*
                                 * When parent is not a partition root or is
                                 * invalid, child partition roots become
                                 * invalid too.
                                 */
                                if (is_partition_valid(cp))
                                        new_prs = -cp->partition_root_state;
                                WRITE_ONCE(cp->prs_err,
                                           is_partition_invalid(parent)
                                           ? PERR_INVPARENT : PERR_NOTPART);
                                break;
                        }
                }
get_css:
                if (!css_tryget_online(&cp->css))
                        continue;
                rcu_read_unlock();

                if (update_parent) {
                        update_parent_effective_cpumask(cp, partcmd_update, NULL, tmp);
                        /*
                         * The cpuset partition_root_state may become
                         * invalid. Capture it.
                         */
                        new_prs = cp->partition_root_state;
                }

                spin_lock_irq(&callback_lock);
                cpumask_copy(cp->effective_cpus, tmp->new_cpus);
                cp->partition_root_state = new_prs;
                if (!cpumask_empty(cp->exclusive_cpus) && (cp != cs))
                        compute_effective_exclusive_cpumask(cp, NULL, NULL);

                /*
                 * Make sure effective_xcpus is properly set for a valid
                 * partition root.
                 */
                if ((new_prs > 0) && cpumask_empty(cp->exclusive_cpus))
                        cpumask_and(cp->effective_xcpus,
                                    cp->cpus_allowed, parent->effective_xcpus);
                else if (new_prs < 0)
                        reset_partition_data(cp);
                spin_unlock_irq(&callback_lock);

                notify_partition_change(cp, old_prs);

                WARN_ON(!is_in_v2_mode() &&
                        !cpumask_equal(cp->cpus_allowed, cp->effective_cpus));

                cpuset_update_tasks_cpumask(cp, cp->effective_cpus);

                /*
                 * On default hierarchy, inherit the CS_SCHED_LOAD_BALANCE
                 * from parent if current cpuset isn't a valid partition root
                 * and their load balance states differ.
                 */
                if (cpuset_v2() && !is_partition_valid(cp) &&
                    (is_sched_load_balance(parent) != is_sched_load_balance(cp))) {
                        if (is_sched_load_balance(parent))
                                set_bit(CS_SCHED_LOAD_BALANCE, &cp->flags);
                        else
                                clear_bit(CS_SCHED_LOAD_BALANCE, &cp->flags);
                }

                /*
                 * On legacy hierarchy, if the effective cpumask of any non-
                 * empty cpuset is changed, we need to rebuild sched domains.
                 * On default hierarchy, the cpuset needs to be a partition
                 * root as well.
                 */
                if (!cpumask_empty(cp->cpus_allowed) &&
                    is_sched_load_balance(cp) &&
                   (!cpuset_v2() || is_partition_valid(cp)))
                        need_rebuild_sched_domains = true;

                rcu_read_lock();
                css_put(&cp->css);
        }
        rcu_read_unlock();

        if (need_rebuild_sched_domains)
                cpuset_force_rebuild();
}

/**
 * update_sibling_cpumasks - Update siblings cpumasks
 * @parent:  Parent cpuset
 * @cs:      Current cpuset
 * @tmp:     Temp variables
 */
static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs,
                                    struct tmpmasks *tmp)
{
        struct cpuset *sibling;
        struct cgroup_subsys_state *pos_css;

        lockdep_assert_held(&cpuset_mutex);

        /*
         * Check all its siblings and call update_cpumasks_hier()
         * if their effective_cpus will need to be changed.
         *
         * It is possible a change in parent's effective_cpus
         * due to a change in a child partition's effective_xcpus will impact
         * its siblings even if they do not inherit parent's effective_cpus
         * directly.
         *
         * The update_cpumasks_hier() function may sleep. So we have to
         * release the RCU read lock before calling it.
         */
        rcu_read_lock();
        cpuset_for_each_child(sibling, pos_css, parent) {
                if (sibling == cs)
                        continue;
                if (!is_partition_valid(sibling)) {
                        compute_effective_cpumask(tmp->new_cpus, sibling,
                                                  parent);
                        if (cpumask_equal(tmp->new_cpus, sibling->effective_cpus))
                                continue;
                } else if (is_remote_partition(sibling)) {
                        /*
                         * Change in a sibling cpuset won't affect a remote
                         * partition root.
                         */
                        continue;
                }

                if (!css_tryget_online(&sibling->css))
                        continue;

                rcu_read_unlock();
                update_cpumasks_hier(sibling, tmp, false);
                rcu_read_lock();
                css_put(&sibling->css);
        }
        rcu_read_unlock();
}

/**
 * update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it
 * @cs: the cpuset to consider
 * @trialcs: trial cpuset
 * @buf: buffer of cpu numbers written to this cpuset
 */
static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs,
                          const char *buf)
{
        int retval;
        struct tmpmasks tmp;
        struct cpuset *parent = parent_cs(cs);
        bool invalidate = false;
        bool force = false;
        int old_prs = cs->partition_root_state;

        /* top_cpuset.cpus_allowed tracks cpu_online_mask; it's read-only */
        if (cs == &top_cpuset)
                return -EACCES;

        /*
         * An empty cpus_allowed is ok only if the cpuset has no tasks.
         * Since cpulist_parse() fails on an empty mask, we special case
         * that parsing.  The validate_change() call ensures that cpusets
         * with tasks have cpus.
         */
        if (!*buf) {
                cpumask_clear(trialcs->cpus_allowed);
                if (cpumask_empty(trialcs->exclusive_cpus))
                        cpumask_clear(trialcs->effective_xcpus);
        } else {
                retval = cpulist_parse(buf, trialcs->cpus_allowed);
                if (retval < 0)
                        return retval;

                if (!cpumask_subset(trialcs->cpus_allowed,
                                    top_cpuset.cpus_allowed))
                        return -EINVAL;

                /*
                 * When exclusive_cpus isn't explicitly set, it is constrained
                 * by cpus_allowed and parent's effective_xcpus. Otherwise,
                 * trialcs->effective_xcpus is used as a temporary cpumask
                 * for checking validity of the partition root.
                 */
                trialcs->partition_root_state = PRS_MEMBER;
                if (!cpumask_empty(trialcs->exclusive_cpus) || is_partition_valid(cs))
                        compute_effective_exclusive_cpumask(trialcs, NULL, cs);
        }

        /* Nothing to do if the cpus didn't change */
        if (cpumask_equal(cs->cpus_allowed, trialcs->cpus_allowed))
                return 0;

        if (alloc_cpumasks(NULL, &tmp))
                return -ENOMEM;

        if (old_prs) {
                if (is_partition_valid(cs) &&
                    cpumask_empty(trialcs->effective_xcpus)) {
                        invalidate = true;
                        cs->prs_err = PERR_INVCPUS;
                } else if (prstate_housekeeping_conflict(old_prs, trialcs->effective_xcpus)) {
                        invalidate = true;
                        cs->prs_err = PERR_HKEEPING;
                } else if (tasks_nocpu_error(parent, cs, trialcs->effective_xcpus)) {
                        invalidate = true;
                        cs->prs_err = PERR_NOCPUS;
                }
        }

        /*
         * Check all the descendants in update_cpumasks_hier() if
         * effective_xcpus is to be changed.
         */
        force = !cpumask_equal(cs->effective_xcpus, trialcs->effective_xcpus);

        retval = validate_change(cs, trialcs);

        if ((retval == -EINVAL) && cpuset_v2()) {
                struct cgroup_subsys_state *css;
                struct cpuset *cp;

                /*
                 * The -EINVAL error code indicates that partition sibling
                 * CPU exclusivity rule has been violated. We still allow
                 * the cpumask change to proceed while invalidating the
                 * partition. However, any conflicting sibling partitions
                 * have to be marked as invalid too.
                 */
                invalidate = true;
                rcu_read_lock();
                cpuset_for_each_child(cp, css, parent) {
                        struct cpumask *xcpus = user_xcpus(trialcs);

                        if (is_partition_valid(cp) &&
                            cpumask_intersects(xcpus, cp->effective_xcpus)) {
                                rcu_read_unlock();
                                update_parent_effective_cpumask(cp, partcmd_invalidate, NULL, &tmp);
                                rcu_read_lock();
                        }
                }
                rcu_read_unlock();
                retval = 0;
        }

        if (retval < 0)
                goto out_free;

        if (is_partition_valid(cs) ||
           (is_partition_invalid(cs) && !invalidate)) {
                struct cpumask *xcpus = trialcs->effective_xcpus;

                if (cpumask_empty(xcpus) && is_partition_invalid(cs))
                        xcpus = trialcs->cpus_allowed;

                /*
                 * Call remote_cpus_update() to handle valid remote partition
                 */
                if (is_remote_partition(cs))
                        remote_cpus_update(cs, NULL, xcpus, &tmp);
                else if (invalidate)
                        update_parent_effective_cpumask(cs, partcmd_invalidate,
                                                        NULL, &tmp);
                else
                        update_parent_effective_cpumask(cs, partcmd_update,
                                                        xcpus, &tmp);
        }

        spin_lock_irq(&callback_lock);
        cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed);
        cpumask_copy(cs->effective_xcpus, trialcs->effective_xcpus);
        if ((old_prs > 0) && !is_partition_valid(cs))
                reset_partition_data(cs);
        spin_unlock_irq(&callback_lock);

        /* effective_cpus/effective_xcpus will be updated here */
        update_cpumasks_hier(cs, &tmp, force);

        /* Update CS_SCHED_LOAD_BALANCE and/or sched_domains, if necessary */
        if (cs->partition_root_state)
                update_partition_sd_lb(cs, old_prs);
out_free:
        free_cpumasks(NULL, &tmp);
        return retval;
}

/**
 * update_exclusive_cpumask - update the exclusive_cpus mask of a cpuset
 * @cs: the cpuset to consider
 * @trialcs: trial cpuset
 * @buf: buffer of cpu numbers written to this cpuset
 *
 * The tasks' cpumask will be updated if cs is a valid partition root.
 */
static int update_exclusive_cpumask(struct cpuset *cs, struct cpuset *trialcs,
                                    const char *buf)
{
        int retval;
        struct tmpmasks tmp;
        struct cpuset *parent = parent_cs(cs);
        bool invalidate = false;
        bool force = false;
        int old_prs = cs->partition_root_state;

        if (!*buf) {
                cpumask_clear(trialcs->exclusive_cpus);
                cpumask_clear(trialcs->effective_xcpus);
        } else {
                retval = cpulist_parse(buf, trialcs->exclusive_cpus);
                if (retval < 0)
                        return retval;
        }

        /* Nothing to do if the CPUs didn't change */
        if (cpumask_equal(cs->exclusive_cpus, trialcs->exclusive_cpus))
                return 0;

        if (*buf) {
                trialcs->partition_root_state = PRS_MEMBER;
                /*
                 * Reject the change if there is exclusive CPUs conflict with
                 * the siblings.
                 */
                if (compute_effective_exclusive_cpumask(trialcs, NULL, cs))
                        return -EINVAL;
        }

        /*
         * Check all the descendants in update_cpumasks_hier() if
         * effective_xcpus is to be changed.
         */
        force = !cpumask_equal(cs->effective_xcpus, trialcs->effective_xcpus);

        retval = validate_change(cs, trialcs);
        if (retval)
                return retval;

        if (alloc_cpumasks(NULL, &tmp))
                return -ENOMEM;

        if (old_prs) {
                if (cpumask_empty(trialcs->effective_xcpus)) {
                        invalidate = true;
                        cs->prs_err = PERR_INVCPUS;
                } else if (prstate_housekeeping_conflict(old_prs, trialcs->effective_xcpus)) {
                        invalidate = true;
                        cs->prs_err = PERR_HKEEPING;
                } else if (tasks_nocpu_error(parent, cs, trialcs->effective_xcpus)) {
                        invalidate = true;
                        cs->prs_err = PERR_NOCPUS;
                }

                if (is_remote_partition(cs)) {
                        if (invalidate)
                                remote_partition_disable(cs, &tmp);
                        else
                                remote_cpus_update(cs, trialcs->exclusive_cpus,
                                                   trialcs->effective_xcpus, &tmp);
                } else if (invalidate) {
                        update_parent_effective_cpumask(cs, partcmd_invalidate,
                                                        NULL, &tmp);
                } else {
                        update_parent_effective_cpumask(cs, partcmd_update,
                                                trialcs->effective_xcpus, &tmp);
                }
        }
        spin_lock_irq(&callback_lock);
        cpumask_copy(cs->exclusive_cpus, trialcs->exclusive_cpus);
        cpumask_copy(cs->effective_xcpus, trialcs->effective_xcpus);
        if ((old_prs > 0) && !is_partition_valid(cs))
                reset_partition_data(cs);
        spin_unlock_irq(&callback_lock);

        /*
         * Call update_cpumasks_hier() to update effective_cpus/effective_xcpus
         * of the subtree when it is a valid partition root or effective_xcpus
         * is updated.
         */
        if (is_partition_valid(cs) || force)
                update_cpumasks_hier(cs, &tmp, force);

        /* Update CS_SCHED_LOAD_BALANCE and/or sched_domains, if necessary */
        if (cs->partition_root_state)
                update_partition_sd_lb(cs, old_prs);

        free_cpumasks(NULL, &tmp);
        return 0;
}

/*
 * Migrate memory region from one set of nodes to another.  This is
 * performed asynchronously as it can be called from process migration path
 * holding locks involved in process management.  All mm migrations are
 * performed in the queued order and can be waited for by flushing
 * cpuset_migrate_mm_wq.
 */

struct cpuset_migrate_mm_work {
        struct work_struct        work;
        struct mm_struct        *mm;
        nodemask_t                from;
        nodemask_t                to;
};

static void cpuset_migrate_mm_workfn(struct work_struct *work)
{
        struct cpuset_migrate_mm_work *mwork =
                container_of(work, struct cpuset_migrate_mm_work, work);

        /* on a wq worker, no need to worry about %current's mems_allowed */
        do_migrate_pages(mwork->mm, &mwork->from, &mwork->to, MPOL_MF_MOVE_ALL);
        mmput(mwork->mm);
        kfree(mwork);
}

static void cpuset_migrate_mm(struct mm_struct *mm, const nodemask_t *from,
                                                        const nodemask_t *to)
{
        struct cpuset_migrate_mm_work *mwork;

        if (nodes_equal(*from, *to)) {
                mmput(mm);
                return;
        }

        mwork = kzalloc(sizeof(*mwork), GFP_KERNEL);
        if (mwork) {
                mwork->mm = mm;
                mwork->from = *from;
                mwork->to = *to;
                INIT_WORK(&mwork->work, cpuset_migrate_mm_workfn);
                queue_work(cpuset_migrate_mm_wq, &mwork->work);
        } else {
                mmput(mm);
        }
}

static void cpuset_post_attach(void)
{
        flush_workqueue(cpuset_migrate_mm_wq);
}

/*
 * cpuset_change_task_nodemask - change task's mems_allowed and mempolicy
 * @tsk: the task to change
 * @newmems: new nodes that the task will be set
 *
 * We use the mems_allowed_seq seqlock to safely update both tsk->mems_allowed
 * and rebind an eventual tasks' mempolicy. If the task is allocating in
 * parallel, it might temporarily see an empty intersection, which results in
 * a seqlock check and retry before OOM or allocation failure.
 */
static void cpuset_change_task_nodemask(struct task_struct *tsk,
                                        nodemask_t *newmems)
{
        task_lock(tsk);

        local_irq_disable();
        write_seqcount_begin(&tsk->mems_allowed_seq);

        nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems);
        mpol_rebind_task(tsk, newmems);
        tsk->mems_allowed = *newmems;

        write_seqcount_end(&tsk->mems_allowed_seq);
        local_irq_enable();

        task_unlock(tsk);
}

static void *cpuset_being_rebound;

/**
 * cpuset_update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.
 * @cs: the cpuset in which each task's mems_allowed mask needs to be changed
 *
 * Iterate through each task of @cs updating its mems_allowed to the
 * effective cpuset's.  As this function is called with cpuset_mutex held,
 * cpuset membership stays stable.
 */
void cpuset_update_tasks_nodemask(struct cpuset *cs)
{
        static nodemask_t newmems;        /* protected by cpuset_mutex */
        struct css_task_iter it;
        struct task_struct *task;

        cpuset_being_rebound = cs;                /* causes mpol_dup() rebind */

        guarantee_online_mems(cs, &newmems);

        /*
         * The mpol_rebind_mm() call takes mmap_lock, which we couldn't
         * take while holding tasklist_lock.  Forks can happen - the
         * mpol_dup() cpuset_being_rebound check will catch such forks,
         * and rebind their vma mempolicies too.  Because we still hold
         * the global cpuset_mutex, we know that no other rebind effort
         * will be contending for the global variable cpuset_being_rebound.
         * It's ok if we rebind the same mm twice; mpol_rebind_mm()
         * is idempotent.  Also migrate pages in each mm to new nodes.
         */
        css_task_iter_start(&cs->css, 0, &it);
        while ((task = css_task_iter_next(&it))) {
                struct mm_struct *mm;
                bool migrate;

                cpuset_change_task_nodemask(task, &newmems);

                mm = get_task_mm(task);
                if (!mm)
                        continue;

                migrate = is_memory_migrate(cs);

                mpol_rebind_mm(mm, &cs->mems_allowed);
                if (migrate)
                        cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems);
                else
                        mmput(mm);
        }
        css_task_iter_end(&it);

        /*
         * All the tasks' nodemasks have been updated, update
         * cs->old_mems_allowed.
         */
        cs->old_mems_allowed = newmems;

        /* We're done rebinding vmas to this cpuset's new mems_allowed. */
        cpuset_being_rebound = NULL;
}

/*
 * update_nodemasks_hier - Update effective nodemasks and tasks in the subtree
 * @cs: the cpuset to consider
 * @new_mems: a temp variable for calculating new effective_mems
 *
 * When configured nodemask is changed, the effective nodemasks of this cpuset
 * and all its descendants need to be updated.
 *
 * On legacy hierarchy, effective_mems will be the same with mems_allowed.
 *
 * Called with cpuset_mutex held
 */
static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems)
{
        struct cpuset *cp;
        struct cgroup_subsys_state *pos_css;

        rcu_read_lock();
        cpuset_for_each_descendant_pre(cp, pos_css, cs) {
                struct cpuset *parent = parent_cs(cp);

                nodes_and(*new_mems, cp->mems_allowed, parent->effective_mems);

                /*
                 * If it becomes empty, inherit the effective mask of the
                 * parent, which is guaranteed to have some MEMs.
                 */
                if (is_in_v2_mode() && nodes_empty(*new_mems))
                        *new_mems = parent->effective_mems;

                /* Skip the whole subtree if the nodemask remains the same. */
                if (nodes_equal(*new_mems, cp->effective_mems)) {
                        pos_css = css_rightmost_descendant(pos_css);
                        continue;
                }

                if (!css_tryget_online(&cp->css))
                        continue;
                rcu_read_unlock();

                spin_lock_irq(&callback_lock);
                cp->effective_mems = *new_mems;
                spin_unlock_irq(&callback_lock);

                WARN_ON(!is_in_v2_mode() &&
                        !nodes_equal(cp->mems_allowed, cp->effective_mems));

                cpuset_update_tasks_nodemask(cp);

                rcu_read_lock();
                css_put(&cp->css);
        }
        rcu_read_unlock();
}

/*
 * Handle user request to change the 'mems' memory placement
 * of a cpuset.  Needs to validate the request, update the
 * cpusets mems_allowed, and for each task in the cpuset,
 * update mems_allowed and rebind task's mempolicy and any vma
 * mempolicies and if the cpuset is marked 'memory_migrate',
 * migrate the tasks pages to the new memory.
 *
 * Call with cpuset_mutex held. May take callback_lock during call.
 * Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
 * lock each such tasks mm->mmap_lock, scan its vma's and rebind
 * their mempolicies to the cpusets new mems_allowed.
 */
static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs,
                           const char *buf)
{
        int retval;

        /*
         * top_cpuset.mems_allowed tracks node_stats[N_MEMORY];
         * it's read-only
         */
        if (cs == &top_cpuset) {
                retval = -EACCES;
                goto done;
        }

        /*
         * An empty mems_allowed is ok iff there are no tasks in the cpuset.
         * Since nodelist_parse() fails on an empty mask, we special case
         * that parsing.  The validate_change() call ensures that cpusets
         * with tasks have memory.
         */
        if (!*buf) {
                nodes_clear(trialcs->mems_allowed);
        } else {
                retval = nodelist_parse(buf, trialcs->mems_allowed);
                if (retval < 0)
                        goto done;

                if (!nodes_subset(trialcs->mems_allowed,
                                  top_cpuset.mems_allowed)) {
                        retval = -EINVAL;
                        goto done;
                }
        }

        if (nodes_equal(cs->mems_allowed, trialcs->mems_allowed)) {
                retval = 0;                /* Too easy - nothing to do */
                goto done;
        }
        retval = validate_change(cs, trialcs);
        if (retval < 0)
                goto done;

        check_insane_mems_config(&trialcs->mems_allowed);

        spin_lock_irq(&callback_lock);
        cs->mems_allowed = trialcs->mems_allowed;
        spin_unlock_irq(&callback_lock);

        /* use trialcs->mems_allowed as a temp variable */
        update_nodemasks_hier(cs, &trialcs->mems_allowed);
done:
        return retval;
}

bool current_cpuset_is_being_rebound(void)
{
        bool ret;

        rcu_read_lock();
        ret = task_cs(current) == cpuset_being_rebound;
        rcu_read_unlock();

        return ret;
}

/*
 * cpuset_update_flag - read a 0 or a 1 in a file and update associated flag
 * bit:                the bit to update (see cpuset_flagbits_t)
 * cs:                the cpuset to update
 * turning_on:         whether the flag is being set or cleared
 *
 * Call with cpuset_mutex held.
 */

int cpuset_update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
                       int turning_on)
{
        struct cpuset *trialcs;
        int balance_flag_changed;
        int spread_flag_changed;
        int err;

        trialcs = alloc_trial_cpuset(cs);
        if (!trialcs)
                return -ENOMEM;

        if (turning_on)
                set_bit(bit, &trialcs->flags);
        else
                clear_bit(bit, &trialcs->flags);

        err = validate_change(cs, trialcs);
        if (err < 0)
                goto out;

        balance_flag_changed = (is_sched_load_balance(cs) !=
                                is_sched_load_balance(trialcs));

        spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs))
                        || (is_spread_page(cs) != is_spread_page(trialcs)));

        spin_lock_irq(&callback_lock);
        cs->flags = trialcs->flags;
        spin_unlock_irq(&callback_lock);

        if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed) {
                if (cpuset_v2())
                        cpuset_force_rebuild();
                else
                        rebuild_sched_domains_locked();
        }

        if (spread_flag_changed)
                cpuset1_update_tasks_flags(cs);
out:
        free_cpuset(trialcs);
        return err;
}

/**
 * update_prstate - update partition_root_state
 * @cs: the cpuset to update
 * @new_prs: new partition root state
 * Return: 0 if successful, != 0 if error
 *
 * Call with cpuset_mutex held.
 */
static int update_prstate(struct cpuset *cs, int new_prs)
{
        int err = PERR_NONE, old_prs = cs->partition_root_state;
        struct cpuset *parent = parent_cs(cs);
        struct tmpmasks tmpmask;
        bool isolcpus_updated = false;

        if (old_prs == new_prs)
                return 0;

        /*
         * Treat a previously invalid partition root as if it is a "member".
         */
        if (new_prs && is_prs_invalid(old_prs))
                old_prs = PRS_MEMBER;

        if (alloc_cpumasks(NULL, &tmpmask))
                return -ENOMEM;

        err = update_partition_exclusive_flag(cs, new_prs);
        if (err)
                goto out;

        if (!old_prs) {
                /*
                 * cpus_allowed and exclusive_cpus cannot be both empty.
                 */
                if (xcpus_empty(cs)) {
                        err = PERR_CPUSEMPTY;
                        goto out;
                }

                /*
                 * We don't support the creation of a new local partition with
                 * a remote partition underneath it. This unsupported
                 * setting can happen only if parent is the top_cpuset because
                 * a remote partition cannot be created underneath an existing
                 * local or remote partition.
                 */
                if ((parent == &top_cpuset) &&
                    cpumask_intersects(cs->exclusive_cpus, subpartitions_cpus)) {
                        err = PERR_REMOTE;
                        goto out;
                }

                /*
                 * If parent is valid partition, enable local partiion.
                 * Otherwise, enable a remote partition.
                 */
                if (is_partition_valid(parent)) {
                        enum partition_cmd cmd = (new_prs == PRS_ROOT)
                                               ? partcmd_enable : partcmd_enablei;

                        err = update_parent_effective_cpumask(cs, cmd, NULL, &tmpmask);
                } else {
                        err = remote_partition_enable(cs, new_prs, &tmpmask);
                }
        } else if (old_prs && new_prs) {
                /*
                 * A change in load balance state only, no change in cpumasks.
                 * Need to update isolated_cpus.
                 */
                isolcpus_updated = true;
        } else {
                /*
                 * Switching back to member is always allowed even if it
                 * disables child partitions.
                 */
                if (is_remote_partition(cs))
                        remote_partition_disable(cs, &tmpmask);
                else
                        update_parent_effective_cpumask(cs, partcmd_disable,
                                                        NULL, &tmpmask);

                /*
                 * Invalidation of child partitions will be done in
                 * update_cpumasks_hier().
                 */
        }
out:
        /*
         * Make partition invalid & disable CS_CPU_EXCLUSIVE if an error
         * happens.
         */
        if (err) {
                new_prs = -new_prs;
                update_partition_exclusive_flag(cs, new_prs);
        }

        spin_lock_irq(&callback_lock);
        cs->partition_root_state = new_prs;
        WRITE_ONCE(cs->prs_err, err);
        if (!is_partition_valid(cs))
                reset_partition_data(cs);
        else if (isolcpus_updated)
                isolated_cpus_update(old_prs, new_prs, cs->effective_xcpus);
        spin_unlock_irq(&callback_lock);
        update_unbound_workqueue_cpumask(isolcpus_updated);

        /* Force update if switching back to member & update effective_xcpus */
        update_cpumasks_hier(cs, &tmpmask, !new_prs);

        /* A newly created partition must have effective_xcpus set */
        WARN_ON_ONCE(!old_prs && (new_prs > 0)
                              && cpumask_empty(cs->effective_xcpus));

        /* Update sched domains and load balance flag */
        update_partition_sd_lb(cs, old_prs);

        notify_partition_change(cs, old_prs);
        if (force_sd_rebuild)
                rebuild_sched_domains_locked();
        free_cpumasks(NULL, &tmpmask);
        return 0;
}

static struct cpuset *cpuset_attach_old_cs;

/*
 * Check to see if a cpuset can accept a new task
 * For v1, cpus_allowed and mems_allowed can't be empty.
 * For v2, effective_cpus can't be empty.
 * Note that in v1, effective_cpus = cpus_allowed.
 */
static int cpuset_can_attach_check(struct cpuset *cs)
{
        if (cpumask_empty(cs->effective_cpus) ||
           (!is_in_v2_mode() && nodes_empty(cs->mems_allowed)))
                return -ENOSPC;
        return 0;
}

static void reset_migrate_dl_data(struct cpuset *cs)
{
        cs->nr_migrate_dl_tasks = 0;
        cs->sum_migrate_dl_bw = 0;
}

/* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */
static int cpuset_can_attach(struct cgroup_taskset *tset)
{
        struct cgroup_subsys_state *css;
        struct cpuset *cs, *oldcs;
        struct task_struct *task;
        bool cpus_updated, mems_updated;
        int ret;

        /* used later by cpuset_attach() */
        cpuset_attach_old_cs = task_cs(cgroup_taskset_first(tset, &css));
        oldcs = cpuset_attach_old_cs;
        cs = css_cs(css);

        mutex_lock(&cpuset_mutex);

        /* Check to see if task is allowed in the cpuset */
        ret = cpuset_can_attach_check(cs);
        if (ret)
                goto out_unlock;

        cpus_updated = !cpumask_equal(cs->effective_cpus, oldcs->effective_cpus);
        mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems);

        cgroup_taskset_for_each(task, css, tset) {
                ret = task_can_attach(task);
                if (ret)
                        goto out_unlock;

                /*
                 * Skip rights over task check in v2 when nothing changes,
                 * migration permission derives from hierarchy ownership in
                 * cgroup_procs_write_permission()).
                 */
                if (!cpuset_v2() || (cpus_updated || mems_updated)) {
                        ret = security_task_setscheduler(task);
                        if (ret)
                                goto out_unlock;
                }

                if (dl_task(task)) {
                        cs->nr_migrate_dl_tasks++;
                        cs->sum_migrate_dl_bw += task->dl.dl_bw;
                }
        }

        if (!cs->nr_migrate_dl_tasks)
                goto out_success;

        if (!cpumask_intersects(oldcs->effective_cpus, cs->effective_cpus)) {
                int cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus);

                if (unlikely(cpu >= nr_cpu_ids)) {
                        reset_migrate_dl_data(cs);
                        ret = -EINVAL;
                        goto out_unlock;
                }

                ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw);
                if (ret) {
                        reset_migrate_dl_data(cs);
                        goto out_unlock;
                }
        }

out_success:
        /*
         * Mark attach is in progress.  This makes validate_change() fail
         * changes which zero cpus/mems_allowed.
         */
        cs->attach_in_progress++;
out_unlock:
        mutex_unlock(&cpuset_mutex);
        return ret;
}

static void cpuset_cancel_attach(struct cgroup_taskset *tset)
{
        struct cgroup_subsys_state *css;
        struct cpuset *cs;

        cgroup_taskset_first(tset, &css);
        cs = css_cs(css);

        mutex_lock(&cpuset_mutex);
        dec_attach_in_progress_locked(cs);

        if (cs->nr_migrate_dl_tasks) {
                int cpu = cpumask_any(cs->effective_cpus);

                dl_bw_free(cpu, cs->sum_migrate_dl_bw);
                reset_migrate_dl_data(cs);
        }

        mutex_unlock(&cpuset_mutex);
}

/*
 * Protected by cpuset_mutex. cpus_attach is used only by cpuset_attach_task()
 * but we can't allocate it dynamically there.  Define it global and
 * allocate from cpuset_init().
 */
static cpumask_var_t cpus_attach;
static nodemask_t cpuset_attach_nodemask_to;

static void cpuset_attach_task(struct cpuset *cs, struct task_struct *task)
{
        lockdep_assert_held(&cpuset_mutex);

        if (cs != &top_cpuset)
                guarantee_online_cpus(task, cpus_attach);
        else
                cpumask_andnot(cpus_attach, task_cpu_possible_mask(task),
                               subpartitions_cpus);
        /*
         * can_attach beforehand should guarantee that this doesn't
         * fail.  TODO: have a better way to handle failure here
         */
        WARN_ON_ONCE(set_cpus_allowed_ptr(task, cpus_attach));

        cpuset_change_task_nodemask(task, &cpuset_attach_nodemask_to);
        cpuset1_update_task_spread_flags(cs, task);
}

static void cpuset_attach(struct cgroup_taskset *tset)
{
        struct task_struct *task;
        struct task_struct *leader;
        struct cgroup_subsys_state *css;
        struct cpuset *cs;
        struct cpuset *oldcs = cpuset_attach_old_cs;
        bool cpus_updated, mems_updated;

        cgroup_taskset_first(tset, &css);
        cs = css_cs(css);

        lockdep_assert_cpus_held();        /* see cgroup_attach_lock() */
        mutex_lock(&cpuset_mutex);
        cpus_updated = !cpumask_equal(cs->effective_cpus,
                                      oldcs->effective_cpus);
        mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems);

        /*
         * In the default hierarchy, enabling cpuset in the child cgroups
         * will trigger a number of cpuset_attach() calls with no change
         * in effective cpus and mems. In that case, we can optimize out
         * by skipping the task iteration and update.
         */
        if (cpuset_v2() && !cpus_updated && !mems_updated) {
                cpuset_attach_nodemask_to = cs->effective_mems;
                goto out;
        }

        guarantee_online_mems(cs, &cpuset_attach_nodemask_to);

        cgroup_taskset_for_each(task, css, tset)
                cpuset_attach_task(cs, task);

        /*
         * Change mm for all threadgroup leaders. This is expensive and may
         * sleep and should be moved outside migration path proper. Skip it
         * if there is no change in effective_mems and CS_MEMORY_MIGRATE is
         * not set.
         */
        cpuset_attach_nodemask_to = cs->effective_mems;
        if (!is_memory_migrate(cs) && !mems_updated)
                goto out;

        cgroup_taskset_for_each_leader(leader, css, tset) {
                struct mm_struct *mm = get_task_mm(leader);

                if (mm) {
                        mpol_rebind_mm(mm, &cpuset_attach_nodemask_to);

                        /*
                         * old_mems_allowed is the same with mems_allowed
                         * here, except if this task is being moved
                         * automatically due to hotplug.  In that case
                         * @mems_allowed has been updated and is empty, so
                         * @old_mems_allowed is the right nodesets that we
                         * migrate mm from.
                         */
                        if (is_memory_migrate(cs))
                                cpuset_migrate_mm(mm, &oldcs->old_mems_allowed,
                                                  &cpuset_attach_nodemask_to);
                        else
                                mmput(mm);
                }
        }

out:
        cs->old_mems_allowed = cpuset_attach_nodemask_to;

        if (cs->nr_migrate_dl_tasks) {
                cs->nr_deadline_tasks += cs->nr_migrate_dl_tasks;
                oldcs->nr_deadline_tasks -= cs->nr_migrate_dl_tasks;
                reset_migrate_dl_data(cs);
        }

        dec_attach_in_progress_locked(cs);

        mutex_unlock(&cpuset_mutex);
}

/*
 * Common handling for a write to a "cpus" or "mems" file.
 */
ssize_t cpuset_write_resmask(struct kernfs_open_file *of,
                                    char *buf, size_t nbytes, loff_t off)
{
        struct cpuset *cs = css_cs(of_css(of));
        struct cpuset *trialcs;
        int retval = -ENODEV;

        buf = strstrip(buf);
        cpus_read_lock();
        mutex_lock(&cpuset_mutex);
        if (!is_cpuset_online(cs))
                goto out_unlock;

        trialcs = alloc_trial_cpuset(cs);
        if (!trialcs) {
                retval = -ENOMEM;
                goto out_unlock;
        }

        switch (of_cft(of)->private) {
        case FILE_CPULIST:
                retval = update_cpumask(cs, trialcs, buf);
                break;
        case FILE_EXCLUSIVE_CPULIST:
                retval = update_exclusive_cpumask(cs, trialcs, buf);
                break;
        case FILE_MEMLIST:
                retval = update_nodemask(cs, trialcs, buf);
                break;
        default:
                retval = -EINVAL;
                break;
        }

        free_cpuset(trialcs);
        if (force_sd_rebuild)
                rebuild_sched_domains_locked();
out_unlock:
        mutex_unlock(&cpuset_mutex);
        cpus_read_unlock();
        flush_workqueue(cpuset_migrate_mm_wq);
        return retval ?: nbytes;
}

/*
 * These ascii lists should be read in a single call, by using a user
 * buffer large enough to hold the entire map.  If read in smaller
 * chunks, there is no guarantee of atomicity.  Since the display format
 * used, list of ranges of sequential numbers, is variable length,
 * and since these maps can change value dynamically, one could read
 * gibberish by doing partial reads while a list was changing.
 */
int cpuset_common_seq_show(struct seq_file *sf, void *v)
{
        struct cpuset *cs = css_cs(seq_css(sf));
        cpuset_filetype_t type = seq_cft(sf)->private;
        int ret = 0;

        spin_lock_irq(&callback_lock);

        switch (type) {
        case FILE_CPULIST:
                seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->cpus_allowed));
                break;
        case FILE_MEMLIST:
                seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->mems_allowed));
                break;
        case FILE_EFFECTIVE_CPULIST:
                seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->effective_cpus));
                break;
        case FILE_EFFECTIVE_MEMLIST:
                seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->effective_mems));
                break;
        case FILE_EXCLUSIVE_CPULIST:
                seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->exclusive_cpus));
                break;
        case FILE_EFFECTIVE_XCPULIST:
                seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->effective_xcpus));
                break;
        case FILE_SUBPARTS_CPULIST:
                seq_printf(sf, "%*pbl\n", cpumask_pr_args(subpartitions_cpus));
                break;
        case FILE_ISOLATED_CPULIST:
                seq_printf(sf, "%*pbl\n", cpumask_pr_args(isolated_cpus));
                break;
        default:
                ret = -EINVAL;
        }

        spin_unlock_irq(&callback_lock);
        return ret;
}

static int cpuset_partition_show(struct seq_file *seq, void *v)
{
        struct cpuset *cs = css_cs(seq_css(seq));
        const char *err, *type = NULL;

        switch (cs->partition_root_state) {
        case PRS_ROOT:
                seq_puts(seq, "root\n");
                break;
        case PRS_ISOLATED:
                seq_puts(seq, "isolated\n");
                break;
        case PRS_MEMBER:
                seq_puts(seq, "member\n");
                break;
        case PRS_INVALID_ROOT:
                type = "root";
                fallthrough;
        case PRS_INVALID_ISOLATED:
                if (!type)
                        type = "isolated";
                err = perr_strings[READ_ONCE(cs->prs_err)];
                if (err)
                        seq_printf(seq, "%s invalid (%s)\n", type, err);
                else
                        seq_printf(seq, "%s invalid\n", type);
                break;
        }
        return 0;
}

static ssize_t cpuset_partition_write(struct kernfs_open_file *of, char *buf,
                                     size_t nbytes, loff_t off)
{
        struct cpuset *cs = css_cs(of_css(of));
        int val;
        int retval = -ENODEV;

        buf = strstrip(buf);

        if (!strcmp(buf, "root"))
                val = PRS_ROOT;
        else if (!strcmp(buf, "member"))
                val = PRS_MEMBER;
        else if (!strcmp(buf, "isolated"))
                val = PRS_ISOLATED;
        else
                return -EINVAL;

        css_get(&cs->css);
        cpus_read_lock();
        mutex_lock(&cpuset_mutex);
        if (is_cpuset_online(cs))
                retval = update_prstate(cs, val);
        mutex_unlock(&cpuset_mutex);
        cpus_read_unlock();
        css_put(&cs->css);
        return retval ?: nbytes;
}

/*
 * This is currently a minimal set for the default hierarchy. It can be
 * expanded later on by migrating more features and control files from v1.
 */
static struct cftype dfl_files[] = {
        {
                .name = "cpus",
                .seq_show = cpuset_common_seq_show,
                .write = cpuset_write_resmask,
                .max_write_len = (100U + 6 * NR_CPUS),
                .private = FILE_CPULIST,
                .flags = CFTYPE_NOT_ON_ROOT,
        },

        {
                .name = "mems",
                .seq_show = cpuset_common_seq_show,
                .write = cpuset_write_resmask,
                .max_write_len = (100U + 6 * MAX_NUMNODES),
                .private = FILE_MEMLIST,
                .flags = CFTYPE_NOT_ON_ROOT,
        },

        {
                .name = "cpus.effective",
                .seq_show = cpuset_common_seq_show,
                .private = FILE_EFFECTIVE_CPULIST,
        },

        {
                .name = "mems.effective",
                .seq_show = cpuset_common_seq_show,
                .private = FILE_EFFECTIVE_MEMLIST,
        },

        {
                .name = "cpus.partition",
                .seq_show = cpuset_partition_show,
                .write = cpuset_partition_write,
                .private = FILE_PARTITION_ROOT,
                .flags = CFTYPE_NOT_ON_ROOT,
                .file_offset = offsetof(struct cpuset, partition_file),
        },

        {
                .name = "cpus.exclusive",
                .seq_show = cpuset_common_seq_show,
                .write = cpuset_write_resmask,
                .max_write_len = (100U + 6 * NR_CPUS),
                .private = FILE_EXCLUSIVE_CPULIST,
                .flags = CFTYPE_NOT_ON_ROOT,
        },

        {
                .name = "cpus.exclusive.effective",
                .seq_show = cpuset_common_seq_show,
                .private = FILE_EFFECTIVE_XCPULIST,
                .flags = CFTYPE_NOT_ON_ROOT,
        },

        {
                .name = "cpus.subpartitions",
                .seq_show = cpuset_common_seq_show,
                .private = FILE_SUBPARTS_CPULIST,
                .flags = CFTYPE_ONLY_ON_ROOT | CFTYPE_DEBUG,
        },

        {
                .name = "cpus.isolated",
                .seq_show = cpuset_common_seq_show,
                .private = FILE_ISOLATED_CPULIST,
                .flags = CFTYPE_ONLY_ON_ROOT,
        },

        { }        /* terminate */
};


/**
 * cpuset_css_alloc - Allocate a cpuset css
 * @parent_css: Parent css of the control group that the new cpuset will be
 *              part of
 * Return: cpuset css on success, -ENOMEM on failure.
 *
 * Allocate and initialize a new cpuset css, for non-NULL @parent_css, return
 * top cpuset css otherwise.
 */
static struct cgroup_subsys_state *
cpuset_css_alloc(struct cgroup_subsys_state *parent_css)
{
        struct cpuset *cs;

        if (!parent_css)
                return &top_cpuset.css;

        cs = kzalloc(sizeof(*cs), GFP_KERNEL);
        if (!cs)
                return ERR_PTR(-ENOMEM);

        if (alloc_cpumasks(cs, NULL)) {
                kfree(cs);
                return ERR_PTR(-ENOMEM);
        }

        __set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
        fmeter_init(&cs->fmeter);
        cs->relax_domain_level = -1;
        INIT_LIST_HEAD(&cs->remote_sibling);

        /* Set CS_MEMORY_MIGRATE for default hierarchy */
        if (cpuset_v2())
                __set_bit(CS_MEMORY_MIGRATE, &cs->flags);

        return &cs->css;
}

static int cpuset_css_online(struct cgroup_subsys_state *css)
{
        struct cpuset *cs = css_cs(css);
        struct cpuset *parent = parent_cs(cs);
        struct cpuset *tmp_cs;
        struct cgroup_subsys_state *pos_css;

        if (!parent)
                return 0;

        cpus_read_lock();
        mutex_lock(&cpuset_mutex);

        set_bit(CS_ONLINE, &cs->flags);
        if (is_spread_page(parent))
                set_bit(CS_SPREAD_PAGE, &cs->flags);
        if (is_spread_slab(parent))
                set_bit(CS_SPREAD_SLAB, &cs->flags);
        /*
         * For v2, clear CS_SCHED_LOAD_BALANCE if parent is isolated
         */
        if (cpuset_v2() && !is_sched_load_balance(parent))
                clear_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);

        cpuset_inc();

        spin_lock_irq(&callback_lock);
        if (is_in_v2_mode()) {
                cpumask_copy(cs->effective_cpus, parent->effective_cpus);
                cs->effective_mems = parent->effective_mems;
        }
        spin_unlock_irq(&callback_lock);

        if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags))
                goto out_unlock;

        /*
         * Clone @parent's configuration if CGRP_CPUSET_CLONE_CHILDREN is
         * set.  This flag handling is implemented in cgroup core for
         * historical reasons - the flag may be specified during mount.
         *
         * Currently, if any sibling cpusets have exclusive cpus or mem, we
         * refuse to clone the configuration - thereby refusing the task to
         * be entered, and as a result refusing the sys_unshare() or
         * clone() which initiated it.  If this becomes a problem for some
         * users who wish to allow that scenario, then this could be
         * changed to grant parent->cpus_allowed-sibling_cpus_exclusive
         * (and likewise for mems) to the new cgroup.
         */
        rcu_read_lock();
        cpuset_for_each_child(tmp_cs, pos_css, parent) {
                if (is_mem_exclusive(tmp_cs) || is_cpu_exclusive(tmp_cs)) {
                        rcu_read_unlock();
                        goto out_unlock;
                }
        }
        rcu_read_unlock();

        spin_lock_irq(&callback_lock);
        cs->mems_allowed = parent->mems_allowed;
        cs->effective_mems = parent->mems_allowed;
        cpumask_copy(cs->cpus_allowed, parent->cpus_allowed);
        cpumask_copy(cs->effective_cpus, parent->cpus_allowed);
        spin_unlock_irq(&callback_lock);
out_unlock:
        mutex_unlock(&cpuset_mutex);
        cpus_read_unlock();
        return 0;
}

/*
 * If the cpuset being removed has its flag 'sched_load_balance'
 * enabled, then simulate turning sched_load_balance off, which
 * will call rebuild_sched_domains_locked(). That is not needed
 * in the default hierarchy where only changes in partition
 * will cause repartitioning.
 *
 * If the cpuset has the 'sched.partition' flag enabled, simulate
 * turning 'sched.partition" off.
 */

static void cpuset_css_offline(struct cgroup_subsys_state *css)
{
        struct cpuset *cs = css_cs(css);

        cpus_read_lock();
        mutex_lock(&cpuset_mutex);

        if (!cpuset_v2() && is_sched_load_balance(cs))
                cpuset_update_flag(CS_SCHED_LOAD_BALANCE, cs, 0);

        cpuset_dec();
        clear_bit(CS_ONLINE, &cs->flags);

        mutex_unlock(&cpuset_mutex);
        cpus_read_unlock();
}

static void cpuset_css_killed(struct cgroup_subsys_state *css)
{
        struct cpuset *cs = css_cs(css);

        cpus_read_lock();
        mutex_lock(&cpuset_mutex);

        /* Reset valid partition back to member */
        if (is_partition_valid(cs))
                update_prstate(cs, PRS_MEMBER);

        mutex_unlock(&cpuset_mutex);
        cpus_read_unlock();

}

static void cpuset_css_free(struct cgroup_subsys_state *css)
{
        struct cpuset *cs = css_cs(css);

        free_cpuset(cs);
}

static void cpuset_bind(struct cgroup_subsys_state *root_css)
{
        mutex_lock(&cpuset_mutex);
        spin_lock_irq(&callback_lock);

        if (is_in_v2_mode()) {
                cpumask_copy(top_cpuset.cpus_allowed, cpu_possible_mask);
                cpumask_copy(top_cpuset.effective_xcpus, cpu_possible_mask);
                top_cpuset.mems_allowed = node_possible_map;
        } else {
                cpumask_copy(top_cpuset.cpus_allowed,
                             top_cpuset.effective_cpus);
                top_cpuset.mems_allowed = top_cpuset.effective_mems;
        }

        spin_unlock_irq(&callback_lock);
        mutex_unlock(&cpuset_mutex);
}

/*
 * In case the child is cloned into a cpuset different from its parent,
 * additional checks are done to see if the move is allowed.
 */
static int cpuset_can_fork(struct task_struct *task, struct css_set *cset)
{
        struct cpuset *cs = css_cs(cset->subsys[cpuset_cgrp_id]);
        bool same_cs;
        int ret;

        rcu_read_lock();
        same_cs = (cs == task_cs(current));
        rcu_read_unlock();

        if (same_cs)
                return 0;

        lockdep_assert_held(&cgroup_mutex);
        mutex_lock(&cpuset_mutex);

        /* Check to see if task is allowed in the cpuset */
        ret = cpuset_can_attach_check(cs);
        if (ret)
                goto out_unlock;

        ret = task_can_attach(task);
        if (ret)
                goto out_unlock;

        ret = security_task_setscheduler(task);
        if (ret)
                goto out_unlock;

        /*
         * Mark attach is in progress.  This makes validate_change() fail
         * changes which zero cpus/mems_allowed.
         */
        cs->attach_in_progress++;
out_unlock:
        mutex_unlock(&cpuset_mutex);
        return ret;
}

static void cpuset_cancel_fork(struct task_struct *task, struct css_set *cset)
{
        struct cpuset *cs = css_cs(cset->subsys[cpuset_cgrp_id]);
        bool same_cs;

        rcu_read_lock();
        same_cs = (cs == task_cs(current));
        rcu_read_unlock();

        if (same_cs)
                return;

        dec_attach_in_progress(cs);
}

/*
 * Make sure the new task conform to the current state of its parent,
 * which could have been changed by cpuset just after it inherits the
 * state from the parent and before it sits on the cgroup's task list.
 */
static void cpuset_fork(struct task_struct *task)
{
        struct cpuset *cs;
        bool same_cs;

        rcu_read_lock();
        cs = task_cs(task);
        same_cs = (cs == task_cs(current));
        rcu_read_unlock();

        if (same_cs) {
                if (cs == &top_cpuset)
                        return;

                set_cpus_allowed_ptr(task, current->cpus_ptr);
                task->mems_allowed = current->mems_allowed;
                return;
        }

        /* CLONE_INTO_CGROUP */
        mutex_lock(&cpuset_mutex);
        guarantee_online_mems(cs, &cpuset_attach_nodemask_to);
        cpuset_attach_task(cs, task);

        dec_attach_in_progress_locked(cs);
        mutex_unlock(&cpuset_mutex);
}

struct cgroup_subsys cpuset_cgrp_subsys = {
        .css_alloc        = cpuset_css_alloc,
        .css_online        = cpuset_css_online,
        .css_offline        = cpuset_css_offline,
        .css_killed        = cpuset_css_killed,
        .css_free        = cpuset_css_free,
        .can_attach        = cpuset_can_attach,
        .cancel_attach        = cpuset_cancel_attach,
        .attach                = cpuset_attach,
        .post_attach        = cpuset_post_attach,
        .bind                = cpuset_bind,
        .can_fork        = cpuset_can_fork,
        .cancel_fork        = cpuset_cancel_fork,
        .fork                = cpuset_fork,
#ifdef CONFIG_CPUSETS_V1
        .legacy_cftypes        = cpuset1_files,
#endif
        .dfl_cftypes        = dfl_files,
        .early_init        = true,
        .threaded        = true,
};

/**
 * cpuset_init - initialize cpusets at system boot
 *
 * Description: Initialize top_cpuset
 **/

int __init cpuset_init(void)
{
        BUG_ON(!alloc_cpumask_var(&top_cpuset.cpus_allowed, GFP_KERNEL));
        BUG_ON(!alloc_cpumask_var(&top_cpuset.effective_cpus, GFP_KERNEL));
        BUG_ON(!alloc_cpumask_var(&top_cpuset.effective_xcpus, GFP_KERNEL));
        BUG_ON(!alloc_cpumask_var(&top_cpuset.exclusive_cpus, GFP_KERNEL));
        BUG_ON(!zalloc_cpumask_var(&subpartitions_cpus, GFP_KERNEL));
        BUG_ON(!zalloc_cpumask_var(&isolated_cpus, GFP_KERNEL));

        cpumask_setall(top_cpuset.cpus_allowed);
        nodes_setall(top_cpuset.mems_allowed);
        cpumask_setall(top_cpuset.effective_cpus);
        cpumask_setall(top_cpuset.effective_xcpus);
        cpumask_setall(top_cpuset.exclusive_cpus);
        nodes_setall(top_cpuset.effective_mems);

        fmeter_init(&top_cpuset.fmeter);
        INIT_LIST_HEAD(&remote_children);

        BUG_ON(!alloc_cpumask_var(&cpus_attach, GFP_KERNEL));

        have_boot_isolcpus = housekeeping_enabled(HK_TYPE_DOMAIN);
        if (have_boot_isolcpus) {
                BUG_ON(!alloc_cpumask_var(&boot_hk_cpus, GFP_KERNEL));
                cpumask_copy(boot_hk_cpus, housekeeping_cpumask(HK_TYPE_DOMAIN));
                cpumask_andnot(isolated_cpus, cpu_possible_mask, boot_hk_cpus);
        }

        return 0;
}

static void
hotplug_update_tasks(struct cpuset *cs,
                     struct cpumask *new_cpus, nodemask_t *new_mems,
                     bool cpus_updated, bool mems_updated)
{
        /* A partition root is allowed to have empty effective cpus */
        if (cpumask_empty(new_cpus) && !is_partition_valid(cs))
                cpumask_copy(new_cpus, parent_cs(cs)->effective_cpus);
        if (nodes_empty(*new_mems))
                *new_mems = parent_cs(cs)->effective_mems;

        spin_lock_irq(&callback_lock);
        cpumask_copy(cs->effective_cpus, new_cpus);
        cs->effective_mems = *new_mems;
        spin_unlock_irq(&callback_lock);

        if (cpus_updated)
                cpuset_update_tasks_cpumask(cs, new_cpus);
        if (mems_updated)
                cpuset_update_tasks_nodemask(cs);
}

void cpuset_force_rebuild(void)
{
        force_sd_rebuild = true;
}

/**
 * cpuset_hotplug_update_tasks - update tasks in a cpuset for hotunplug
 * @cs: cpuset in interest
 * @tmp: the tmpmasks structure pointer
 *
 * Compare @cs's cpu and mem masks against top_cpuset and if some have gone
 * offline, update @cs accordingly.  If @cs ends up with no CPU or memory,
 * all its tasks are moved to the nearest ancestor with both resources.
 */
static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp)
{
        static cpumask_t new_cpus;
        static nodemask_t new_mems;
        bool cpus_updated;
        bool mems_updated;
        bool remote;
        int partcmd = -1;
        struct cpuset *parent;
retry:
        wait_event(cpuset_attach_wq, cs->attach_in_progress == 0);

        mutex_lock(&cpuset_mutex);

        /*
         * We have raced with task attaching. We wait until attaching
         * is finished, so we won't attach a task to an empty cpuset.
         */
        if (cs->attach_in_progress) {
                mutex_unlock(&cpuset_mutex);
                goto retry;
        }

        parent = parent_cs(cs);
        compute_effective_cpumask(&new_cpus, cs, parent);
        nodes_and(new_mems, cs->mems_allowed, parent->effective_mems);

        if (!tmp || !cs->partition_root_state)
                goto update_tasks;

        /*
         * Compute effective_cpus for valid partition root, may invalidate
         * child partition roots if necessary.
         */
        remote = is_remote_partition(cs);
        if (remote || (is_partition_valid(cs) && is_partition_valid(parent)))
                compute_partition_effective_cpumask(cs, &new_cpus);

        if (remote && cpumask_empty(&new_cpus) &&
            partition_is_populated(cs, NULL)) {
                cs->prs_err = PERR_HOTPLUG;
                remote_partition_disable(cs, tmp);
                compute_effective_cpumask(&new_cpus, cs, parent);
                remote = false;
        }

        /*
         * Force the partition to become invalid if either one of
         * the following conditions hold:
         * 1) empty effective cpus but not valid empty partition.
         * 2) parent is invalid or doesn't grant any cpus to child
         *    partitions.
         */
        if (is_local_partition(cs) && (!is_partition_valid(parent) ||
                                tasks_nocpu_error(parent, cs, &new_cpus)))
                partcmd = partcmd_invalidate;
        /*
         * On the other hand, an invalid partition root may be transitioned
         * back to a regular one.
         */
        else if (is_partition_valid(parent) && is_partition_invalid(cs))
                partcmd = partcmd_update;

        if (partcmd >= 0) {
                update_parent_effective_cpumask(cs, partcmd, NULL, tmp);
                if ((partcmd == partcmd_invalidate) || is_partition_valid(cs)) {
                        compute_partition_effective_cpumask(cs, &new_cpus);
                        cpuset_force_rebuild();
                }
        }

update_tasks:
        cpus_updated = !cpumask_equal(&new_cpus, cs->effective_cpus);
        mems_updated = !nodes_equal(new_mems, cs->effective_mems);
        if (!cpus_updated && !mems_updated)
                goto unlock;        /* Hotplug doesn't affect this cpuset */

        if (mems_updated)
                check_insane_mems_config(&new_mems);

        if (is_in_v2_mode())
                hotplug_update_tasks(cs, &new_cpus, &new_mems,
                                     cpus_updated, mems_updated);
        else
                cpuset1_hotplug_update_tasks(cs, &new_cpus, &new_mems,
                                            cpus_updated, mems_updated);

unlock:
        mutex_unlock(&cpuset_mutex);
}

/**
 * cpuset_handle_hotplug - handle CPU/memory hot{,un}plug for a cpuset
 *
 * This function is called after either CPU or memory configuration has
 * changed and updates cpuset accordingly.  The top_cpuset is always
 * synchronized to cpu_active_mask and N_MEMORY, which is necessary in
 * order to make cpusets transparent (of no affect) on systems that are
 * actively using CPU hotplug but making no active use of cpusets.
 *
 * Non-root cpusets are only affected by offlining.  If any CPUs or memory
 * nodes have been taken down, cpuset_hotplug_update_tasks() is invoked on
 * all descendants.
 *
 * Note that CPU offlining during suspend is ignored.  We don't modify
 * cpusets across suspend/resume cycles at all.
 *
 * CPU / memory hotplug is handled synchronously.
 */
static void cpuset_handle_hotplug(void)
{
        static cpumask_t new_cpus;
        static nodemask_t new_mems;
        bool cpus_updated, mems_updated;
        bool on_dfl = is_in_v2_mode();
        struct tmpmasks tmp, *ptmp = NULL;

        if (on_dfl && !alloc_cpumasks(NULL, &tmp))
                ptmp = &tmp;

        lockdep_assert_cpus_held();
        mutex_lock(&cpuset_mutex);

        /* fetch the available cpus/mems and find out which changed how */
        cpumask_copy(&new_cpus, cpu_active_mask);
        new_mems = node_states[N_MEMORY];

        /*
         * If subpartitions_cpus is populated, it is likely that the check
         * below will produce a false positive on cpus_updated when the cpu
         * list isn't changed. It is extra work, but it is better to be safe.
         */
        cpus_updated = !cpumask_equal(top_cpuset.effective_cpus, &new_cpus) ||
                       !cpumask_empty(subpartitions_cpus);
        mems_updated = !nodes_equal(top_cpuset.effective_mems, new_mems);

        /* For v1, synchronize cpus_allowed to cpu_active_mask */
        if (cpus_updated) {
                cpuset_force_rebuild();
                spin_lock_irq(&callback_lock);
                if (!on_dfl)
                        cpumask_copy(top_cpuset.cpus_allowed, &new_cpus);
                /*
                 * Make sure that CPUs allocated to child partitions
                 * do not show up in effective_cpus. If no CPU is left,
                 * we clear the subpartitions_cpus & let the child partitions
                 * fight for the CPUs again.
                 */
                if (!cpumask_empty(subpartitions_cpus)) {
                        if (cpumask_subset(&new_cpus, subpartitions_cpus)) {
                                top_cpuset.nr_subparts = 0;
                                cpumask_clear(subpartitions_cpus);
                        } else {
                                cpumask_andnot(&new_cpus, &new_cpus,
                                               subpartitions_cpus);
                        }
                }
                cpumask_copy(top_cpuset.effective_cpus, &new_cpus);
                spin_unlock_irq(&callback_lock);
                /* we don't mess with cpumasks of tasks in top_cpuset */
        }

        /* synchronize mems_allowed to N_MEMORY */
        if (mems_updated) {
                spin_lock_irq(&callback_lock);
                if (!on_dfl)
                        top_cpuset.mems_allowed = new_mems;
                top_cpuset.effective_mems = new_mems;
                spin_unlock_irq(&callback_lock);
                cpuset_update_tasks_nodemask(&top_cpuset);
        }

        mutex_unlock(&cpuset_mutex);

        /* if cpus or mems changed, we need to propagate to descendants */
        if (cpus_updated || mems_updated) {
                struct cpuset *cs;
                struct cgroup_subsys_state *pos_css;

                rcu_read_lock();
                cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
                        if (cs == &top_cpuset || !css_tryget_online(&cs->css))
                                continue;
                        rcu_read_unlock();

                        cpuset_hotplug_update_tasks(cs, ptmp);

                        rcu_read_lock();
                        css_put(&cs->css);
                }
                rcu_read_unlock();
        }

        /* rebuild sched domains if necessary */
        if (force_sd_rebuild)
                rebuild_sched_domains_cpuslocked();

        free_cpumasks(NULL, ptmp);
}

void cpuset_update_active_cpus(void)
{
        /*
         * We're inside cpu hotplug critical region which usually nests
         * inside cgroup synchronization.  Bounce actual hotplug processing
         * to a work item to avoid reverse locking order.
         */
        cpuset_handle_hotplug();
}

/*
 * Keep top_cpuset.mems_allowed tracking node_states[N_MEMORY].
 * Call this routine anytime after node_states[N_MEMORY] changes.
 * See cpuset_update_active_cpus() for CPU hotplug handling.
 */
static int cpuset_track_online_nodes(struct notifier_block *self,
                                unsigned long action, void *arg)
{
        cpuset_handle_hotplug();
        return NOTIFY_OK;
}

/**
 * cpuset_init_smp - initialize cpus_allowed
 *
 * Description: Finish top cpuset after cpu, node maps are initialized
 */
void __init cpuset_init_smp(void)
{
        /*
         * cpus_allowd/mems_allowed set to v2 values in the initial
         * cpuset_bind() call will be reset to v1 values in another
         * cpuset_bind() call when v1 cpuset is mounted.
         */
        top_cpuset.old_mems_allowed = top_cpuset.mems_allowed;

        cpumask_copy(top_cpuset.effective_cpus, cpu_active_mask);
        top_cpuset.effective_mems = node_states[N_MEMORY];

        hotplug_memory_notifier(cpuset_track_online_nodes, CPUSET_CALLBACK_PRI);

        cpuset_migrate_mm_wq = alloc_ordered_workqueue("cpuset_migrate_mm", 0);
        BUG_ON(!cpuset_migrate_mm_wq);
}

/**
 * cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset.
 * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.
 * @pmask: pointer to struct cpumask variable to receive cpus_allowed set.
 *
 * Description: Returns the cpumask_var_t cpus_allowed of the cpuset
 * attached to the specified @tsk.  Guaranteed to return some non-empty
 * subset of cpu_online_mask, even if this means going outside the
 * tasks cpuset, except when the task is in the top cpuset.
 **/

void cpuset_cpus_allowed(struct task_struct *tsk, struct cpumask *pmask)
{
        unsigned long flags;
        struct cpuset *cs;

        spin_lock_irqsave(&callback_lock, flags);
        rcu_read_lock();

        cs = task_cs(tsk);
        if (cs != &top_cpuset)
                guarantee_online_cpus(tsk, pmask);
        /*
         * Tasks in the top cpuset won't get update to their cpumasks
         * when a hotplug online/offline event happens. So we include all
         * offline cpus in the allowed cpu list.
         */
        if ((cs == &top_cpuset) || cpumask_empty(pmask)) {
                const struct cpumask *possible_mask = task_cpu_possible_mask(tsk);

                /*
                 * We first exclude cpus allocated to partitions. If there is no
                 * allowable online cpu left, we fall back to all possible cpus.
                 */
                cpumask_andnot(pmask, possible_mask, subpartitions_cpus);
                if (!cpumask_intersects(pmask, cpu_online_mask))
                        cpumask_copy(pmask, possible_mask);
        }

        rcu_read_unlock();
        spin_unlock_irqrestore(&callback_lock, flags);
}

/**
 * cpuset_cpus_allowed_fallback - final fallback before complete catastrophe.
 * @tsk: pointer to task_struct with which the scheduler is struggling
 *
 * Description: In the case that the scheduler cannot find an allowed cpu in
 * tsk->cpus_allowed, we fall back to task_cs(tsk)->cpus_allowed. In legacy
 * mode however, this value is the same as task_cs(tsk)->effective_cpus,
 * which will not contain a sane cpumask during cases such as cpu hotplugging.
 * This is the absolute last resort for the scheduler and it is only used if
 * _every_ other avenue has been traveled.
 *
 * Returns true if the affinity of @tsk was changed, false otherwise.
 **/

bool cpuset_cpus_allowed_fallback(struct task_struct *tsk)
{
        const struct cpumask *possible_mask = task_cpu_possible_mask(tsk);
        const struct cpumask *cs_mask;
        bool changed = false;

        rcu_read_lock();
        cs_mask = task_cs(tsk)->cpus_allowed;
        if (is_in_v2_mode() && cpumask_subset(cs_mask, possible_mask)) {
                do_set_cpus_allowed(tsk, cs_mask);
                changed = true;
        }
        rcu_read_unlock();

        /*
         * We own tsk->cpus_allowed, nobody can change it under us.
         *
         * But we used cs && cs->cpus_allowed lockless and thus can
         * race with cgroup_attach_task() or update_cpumask() and get
         * the wrong tsk->cpus_allowed. However, both cases imply the
         * subsequent cpuset_change_cpumask()->set_cpus_allowed_ptr()
         * which takes task_rq_lock().
         *
         * If we are called after it dropped the lock we must see all
         * changes in tsk_cs()->cpus_allowed. Otherwise we can temporary
         * set any mask even if it is not right from task_cs() pov,
         * the pending set_cpus_allowed_ptr() will fix things.
         *
         * select_fallback_rq() will fix things ups and set cpu_possible_mask
         * if required.
         */
        return changed;
}

void __init cpuset_init_current_mems_allowed(void)
{
        nodes_setall(current->mems_allowed);
}

/**
 * cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset.
 * @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed.
 *
 * Description: Returns the nodemask_t mems_allowed of the cpuset
 * attached to the specified @tsk.  Guaranteed to return some non-empty
 * subset of node_states[N_MEMORY], even if this means going outside the
 * tasks cpuset.
 **/

nodemask_t cpuset_mems_allowed(struct task_struct *tsk)
{
        nodemask_t mask;
        unsigned long flags;

        spin_lock_irqsave(&callback_lock, flags);
        rcu_read_lock();
        guarantee_online_mems(task_cs(tsk), &mask);
        rcu_read_unlock();
        spin_unlock_irqrestore(&callback_lock, flags);

        return mask;
}

/**
 * cpuset_nodemask_valid_mems_allowed - check nodemask vs. current mems_allowed
 * @nodemask: the nodemask to be checked
 *
 * Are any of the nodes in the nodemask allowed in current->mems_allowed?
 */
int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask)
{
        return nodes_intersects(*nodemask, current->mems_allowed);
}

/*
 * nearest_hardwall_ancestor() - Returns the nearest mem_exclusive or
 * mem_hardwall ancestor to the specified cpuset.  Call holding
 * callback_lock.  If no ancestor is mem_exclusive or mem_hardwall
 * (an unusual configuration), then returns the root cpuset.
 */
static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs)
{
        while (!(is_mem_exclusive(cs) || is_mem_hardwall(cs)) && parent_cs(cs))
                cs = parent_cs(cs);
        return cs;
}

/*
 * cpuset_node_allowed - Can we allocate on a memory node?
 * @node: is this an allowed node?
 * @gfp_mask: memory allocation flags
 *
 * If we're in interrupt, yes, we can always allocate.  If @node is set in
 * current's mems_allowed, yes.  If it's not a __GFP_HARDWALL request and this
 * node is set in the nearest hardwalled cpuset ancestor to current's cpuset,
 * yes.  If current has access to memory reserves as an oom victim, yes.
 * Otherwise, no.
 *
 * GFP_USER allocations are marked with the __GFP_HARDWALL bit,
 * and do not allow allocations outside the current tasks cpuset
 * unless the task has been OOM killed.
 * GFP_KERNEL allocations are not so marked, so can escape to the
 * nearest enclosing hardwalled ancestor cpuset.
 *
 * Scanning up parent cpusets requires callback_lock.  The
 * __alloc_pages() routine only calls here with __GFP_HARDWALL bit
 * _not_ set if it's a GFP_KERNEL allocation, and all nodes in the
 * current tasks mems_allowed came up empty on the first pass over
 * the zonelist.  So only GFP_KERNEL allocations, if all nodes in the
 * cpuset are short of memory, might require taking the callback_lock.
 *
 * The first call here from mm/page_alloc:get_page_from_freelist()
 * has __GFP_HARDWALL set in gfp_mask, enforcing hardwall cpusets,
 * so no allocation on a node outside the cpuset is allowed (unless
 * in interrupt, of course).
 *
 * The second pass through get_page_from_freelist() doesn't even call
 * here for GFP_ATOMIC calls.  For those calls, the __alloc_pages()
 * variable 'wait' is not set, and the bit ALLOC_CPUSET is not set
 * in alloc_flags.  That logic and the checks below have the combined
 * affect that:
 *        in_interrupt - any node ok (current task context irrelevant)
 *        GFP_ATOMIC   - any node ok
 *        tsk_is_oom_victim   - any node ok
 *        GFP_KERNEL   - any node in enclosing hardwalled cpuset ok
 *        GFP_USER     - only nodes in current tasks mems allowed ok.
 */
bool cpuset_node_allowed(int node, gfp_t gfp_mask)
{
        struct cpuset *cs;                /* current cpuset ancestors */
        bool allowed;                        /* is allocation in zone z allowed? */
        unsigned long flags;

        if (in_interrupt())
                return true;
        if (node_isset(node, current->mems_allowed))
                return true;
        /*
         * Allow tasks that have access to memory reserves because they have
         * been OOM killed to get memory anywhere.
         */
        if (unlikely(tsk_is_oom_victim(current)))
                return true;
        if (gfp_mask & __GFP_HARDWALL)        /* If hardwall request, stop here */
                return false;

        if (current->flags & PF_EXITING) /* Let dying task have memory */
                return true;

        /* Not hardwall and node outside mems_allowed: scan up cpusets */
        spin_lock_irqsave(&callback_lock, flags);

        rcu_read_lock();
        cs = nearest_hardwall_ancestor(task_cs(current));
        allowed = node_isset(node, cs->mems_allowed);
        rcu_read_unlock();

        spin_unlock_irqrestore(&callback_lock, flags);
        return allowed;
}

/**
 * cpuset_spread_node() - On which node to begin search for a page
 * @rotor: round robin rotor
 *
 * If a task is marked PF_SPREAD_PAGE or PF_SPREAD_SLAB (as for
 * tasks in a cpuset with is_spread_page or is_spread_slab set),
 * and if the memory allocation used cpuset_mem_spread_node()
 * to determine on which node to start looking, as it will for
 * certain page cache or slab cache pages such as used for file
 * system buffers and inode caches, then instead of starting on the
 * local node to look for a free page, rather spread the starting
 * node around the tasks mems_allowed nodes.
 *
 * We don't have to worry about the returned node being offline
 * because "it can't happen", and even if it did, it would be ok.
 *
 * The routines calling guarantee_online_mems() are careful to
 * only set nodes in task->mems_allowed that are online.  So it
 * should not be possible for the following code to return an
 * offline node.  But if it did, that would be ok, as this routine
 * is not returning the node where the allocation must be, only
 * the node where the search should start.  The zonelist passed to
 * __alloc_pages() will include all nodes.  If the slab allocator
 * is passed an offline node, it will fall back to the local node.
 * See kmem_cache_alloc_node().
 */
static int cpuset_spread_node(int *rotor)
{
        return *rotor = next_node_in(*rotor, current->mems_allowed);
}

/**
 * cpuset_mem_spread_node() - On which node to begin search for a file page
 */
int cpuset_mem_spread_node(void)
{
        if (current->cpuset_mem_spread_rotor == NUMA_NO_NODE)
                current->cpuset_mem_spread_rotor =
                        node_random(&current->mems_allowed);

        return cpuset_spread_node(&current->cpuset_mem_spread_rotor);
}

/**
 * cpuset_mems_allowed_intersects - Does @tsk1's mems_allowed intersect @tsk2's?
 * @tsk1: pointer to task_struct of some task.
 * @tsk2: pointer to task_struct of some other task.
 *
 * Description: Return true if @tsk1's mems_allowed intersects the
 * mems_allowed of @tsk2.  Used by the OOM killer to determine if
 * one of the task's memory usage might impact the memory available
 * to the other.
 **/

int cpuset_mems_allowed_intersects(const struct task_struct *tsk1,
                                   const struct task_struct *tsk2)
{
        return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed);
}

/**
 * cpuset_print_current_mems_allowed - prints current's cpuset and mems_allowed
 *
 * Description: Prints current's name, cpuset name, and cached copy of its
 * mems_allowed to the kernel log.
 */
void cpuset_print_current_mems_allowed(void)
{
        struct cgroup *cgrp;

        rcu_read_lock();

        cgrp = task_cs(current)->css.cgroup;
        pr_cont(",cpuset=");
        pr_cont_cgroup_name(cgrp);
        pr_cont(",mems_allowed=%*pbl",
                nodemask_pr_args(&current->mems_allowed));

        rcu_read_unlock();
}

/* Display task mems_allowed in /proc/<pid>/status file. */
void cpuset_task_status_allowed(struct seq_file *m, struct task_struct *task)
{
        seq_printf(m, "Mems_allowed:\t%*pb\n",
                   nodemask_pr_args(&task->mems_allowed));
        seq_printf(m, "Mems_allowed_list:\t%*pbl\n",
                   nodemask_pr_args(&task->mems_allowed));
}









































































































   69 








   70 





    3 









   70 

















   70 






























   70 

















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  Copyright (C) 2009  Red Hat, Inc.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/sched/numa_balancing.h>
#include <linux/highmem.h>
#include <linux/hugetlb.h>
#include <linux/mmu_notifier.h>
#include <linux/rmap.h>
#include <linux/swap.h>
#include <linux/shrinker.h>
#include <linux/mm_inline.h>
#include <linux/swapops.h>
#include <linux/backing-dev.h>
#include <linux/dax.h>
#include <linux/mm_types.h>
#include <linux/khugepaged.h>
#include <linux/freezer.h>
#include <linux/pfn_t.h>
#include <linux/mman.h>
#include <linux/memremap.h>
#include <linux/pagemap.h>
#include <linux/debugfs.h>
#include <linux/migrate.h>
#include <linux/hashtable.h>
#include <linux/userfaultfd_k.h>
#include <linux/page_idle.h>
#include <linux/shmem_fs.h>
#include <linux/oom.h>
#include <linux/numa.h>
#include <linux/page_owner.h>
#include <linux/sched/sysctl.h>
#include <linux/memory-tiers.h>
#include <linux/compat.h>
#include <linux/pgalloc_tag.h>
#include <linux/pagewalk.h>

#include <asm/tlb.h>
#include <asm/pgalloc.h>
#include "internal.h"
#include "swap.h"

#define CREATE_TRACE_POINTS
#include <trace/events/thp.h>

/*
 * By default, transparent hugepage support is disabled in order to avoid
 * risking an increased memory footprint for applications that are not
 * guaranteed to benefit from it. When transparent hugepage support is
 * enabled, it is for all mappings, and khugepaged scans all mappings.
 * Defrag is invoked by khugepaged hugepage allocations and by page faults
 * for all hugepage allocations.
 */
unsigned long transparent_hugepage_flags __read_mostly =
#ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS
        (1<<TRANSPARENT_HUGEPAGE_FLAG)|
#endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE
        (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)|
#endif
        (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)|
        (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)|
        (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);

static struct shrinker *deferred_split_shrinker;
static unsigned long deferred_split_count(struct shrinker *shrink,
                                          struct shrink_control *sc);
static unsigned long deferred_split_scan(struct shrinker *shrink,
                                         struct shrink_control *sc);
static bool split_underused_thp = true;

static atomic_t huge_zero_refcount;
struct folio *huge_zero_folio __read_mostly;
unsigned long huge_zero_pfn __read_mostly = ~0UL;
unsigned long huge_anon_orders_always __read_mostly;
unsigned long huge_anon_orders_madvise __read_mostly;
unsigned long huge_anon_orders_inherit __read_mostly;
static bool anon_orders_configured __initdata;

static inline bool file_thp_enabled(struct vm_area_struct *vma)
{
        struct inode *inode;

        if (!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS))
                return false;

        if (!vma->vm_file)
                return false;

        inode = file_inode(vma->vm_file);

        return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode);
}

unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
                                         unsigned long vm_flags,
                                         unsigned long tva_flags,
                                         unsigned long orders)
{
        bool smaps = tva_flags & TVA_SMAPS;
        bool in_pf = tva_flags & TVA_IN_PF;
        bool enforce_sysfs = tva_flags & TVA_ENFORCE_SYSFS;
        unsigned long supported_orders;

        /* Check the intersection of requested and supported orders. */
        if (vma_is_anonymous(vma))
                supported_orders = THP_ORDERS_ALL_ANON;
        else if (vma_is_special_huge(vma))
                supported_orders = THP_ORDERS_ALL_SPECIAL;
        else
                supported_orders = THP_ORDERS_ALL_FILE_DEFAULT;

        orders &= supported_orders;
        if (!orders)
                return 0;

        if (!vma->vm_mm)                /* vdso */
                return 0;

        if (thp_disabled_by_hw() || vma_thp_disabled(vma, vm_flags))
                return 0;

        /* khugepaged doesn't collapse DAX vma, but page fault is fine. */
        if (vma_is_dax(vma))
                return in_pf ? orders : 0;

        /*
         * khugepaged special VMA and hugetlb VMA.
         * Must be checked after dax since some dax mappings may have
         * VM_MIXEDMAP set.
         */
        if (!in_pf && !smaps && (vm_flags & VM_NO_KHUGEPAGED))
                return 0;

        /*
         * Check alignment for file vma and size for both file and anon vma by
         * filtering out the unsuitable orders.
         *
         * Skip the check for page fault. Huge fault does the check in fault
         * handlers.
         */
        if (!in_pf) {
                int order = highest_order(orders);
                unsigned long addr;

                while (orders) {
                        addr = vma->vm_end - (PAGE_SIZE << order);
                        if (thp_vma_suitable_order(vma, addr, order))
                                break;
                        order = next_order(&orders, order);
                }

                if (!orders)
                        return 0;
        }

        /*
         * Enabled via shmem mount options or sysfs settings.
         * Must be done before hugepage flags check since shmem has its
         * own flags.
         */
        if (!in_pf && shmem_file(vma->vm_file))
                return shmem_allowable_huge_orders(file_inode(vma->vm_file),
                                                   vma, vma->vm_pgoff, 0,
                                                   !enforce_sysfs);

        if (!vma_is_anonymous(vma)) {
                /*
                 * Enforce sysfs THP requirements as necessary. Anonymous vmas
                 * were already handled in thp_vma_allowable_orders().
                 */
                if (enforce_sysfs &&
                    (!hugepage_global_enabled() || (!(vm_flags & VM_HUGEPAGE) &&
                                                    !hugepage_global_always())))
                        return 0;

                /*
                 * Trust that ->huge_fault() handlers know what they are doing
                 * in fault path.
                 */
                if (((in_pf || smaps)) && vma->vm_ops->huge_fault)
                        return orders;
                /* Only regular file is valid in collapse path */
                if (((!in_pf || smaps)) && file_thp_enabled(vma))
                        return orders;
                return 0;
        }

        if (vma_is_temporary_stack(vma))
                return 0;

        /*
         * THPeligible bit of smaps should show 1 for proper VMAs even
         * though anon_vma is not initialized yet.
         *
         * Allow page fault since anon_vma may be not initialized until
         * the first page fault.
         */
        if (!vma->anon_vma)
                return (smaps || in_pf) ? orders : 0;

        return orders;
}

static bool get_huge_zero_page(void)
{
        struct folio *zero_folio;
retry:
        if (likely(atomic_inc_not_zero(&huge_zero_refcount)))
                return true;

        zero_folio = folio_alloc((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE,
                        HPAGE_PMD_ORDER);
        if (!zero_folio) {
                count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED);
                return false;
        }
        /* Ensure zero folio won't have large_rmappable flag set. */
        folio_clear_large_rmappable(zero_folio);
        preempt_disable();
        if (cmpxchg(&huge_zero_folio, NULL, zero_folio)) {
                preempt_enable();
                folio_put(zero_folio);
                goto retry;
        }
        WRITE_ONCE(huge_zero_pfn, folio_pfn(zero_folio));

        /* We take additional reference here. It will be put back by shrinker */
        atomic_set(&huge_zero_refcount, 2);
        preempt_enable();
        count_vm_event(THP_ZERO_PAGE_ALLOC);
        return true;
}

static void put_huge_zero_page(void)
{
        /*
         * Counter should never go to zero here. Only shrinker can put
         * last reference.
         */
        BUG_ON(atomic_dec_and_test(&huge_zero_refcount));
}

struct folio *mm_get_huge_zero_folio(struct mm_struct *mm)
{
        if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
                return READ_ONCE(huge_zero_folio);

        if (!get_huge_zero_page())
                return NULL;

        if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
                put_huge_zero_page();

        return READ_ONCE(huge_zero_folio);
}

void mm_put_huge_zero_folio(struct mm_struct *mm)
{
        if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
                put_huge_zero_page();
}

static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink,
                                        struct shrink_control *sc)
{
        /* we can free zero page only if last reference remains */
        return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
}

static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink,
                                       struct shrink_control *sc)
{
        if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) {
                struct folio *zero_folio = xchg(&huge_zero_folio, NULL);
                BUG_ON(zero_folio == NULL);
                WRITE_ONCE(huge_zero_pfn, ~0UL);
                folio_put(zero_folio);
                return HPAGE_PMD_NR;
        }

        return 0;
}

static struct shrinker *huge_zero_page_shrinker;

#ifdef CONFIG_SYSFS
static ssize_t enabled_show(struct kobject *kobj,
                            struct kobj_attribute *attr, char *buf)
{
        const char *output;

        if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags))
                output = "[always] madvise never";
        else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
                          &transparent_hugepage_flags))
                output = "always [madvise] never";
        else
                output = "always madvise [never]";

        return sysfs_emit(buf, "%s\n", output);
}

static ssize_t enabled_store(struct kobject *kobj,
                             struct kobj_attribute *attr,
                             const char *buf, size_t count)
{
        ssize_t ret = count;

        if (sysfs_streq(buf, "always")) {
                clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
                set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
        } else if (sysfs_streq(buf, "madvise")) {
                clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
                set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
        } else if (sysfs_streq(buf, "never")) {
                clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
        } else
                ret = -EINVAL;

        if (ret > 0) {
                int err = start_stop_khugepaged();
                if (err)
                        ret = err;
        }
        return ret;
}

static struct kobj_attribute enabled_attr = __ATTR_RW(enabled);

ssize_t single_hugepage_flag_show(struct kobject *kobj,
                                  struct kobj_attribute *attr, char *buf,
                                  enum transparent_hugepage_flag flag)
{
        return sysfs_emit(buf, "%d\n",
                          !!test_bit(flag, &transparent_hugepage_flags));
}

ssize_t single_hugepage_flag_store(struct kobject *kobj,
                                 struct kobj_attribute *attr,
                                 const char *buf, size_t count,
                                 enum transparent_hugepage_flag flag)
{
        unsigned long value;
        int ret;

        ret = kstrtoul(buf, 10, &value);
        if (ret < 0)
                return ret;
        if (value > 1)
                return -EINVAL;

        if (value)
                set_bit(flag, &transparent_hugepage_flags);
        else
                clear_bit(flag, &transparent_hugepage_flags);

        return count;
}

static ssize_t defrag_show(struct kobject *kobj,
                           struct kobj_attribute *attr, char *buf)
{
        const char *output;

        if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
                     &transparent_hugepage_flags))
                output = "[always] defer defer+madvise madvise never";
        else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
                          &transparent_hugepage_flags))
                output = "always [defer] defer+madvise madvise never";
        else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG,
                          &transparent_hugepage_flags))
                output = "always defer [defer+madvise] madvise never";
        else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
                          &transparent_hugepage_flags))
                output = "always defer defer+madvise [madvise] never";
        else
                output = "always defer defer+madvise madvise [never]";

        return sysfs_emit(buf, "%s\n", output);
}

static ssize_t defrag_store(struct kobject *kobj,
                            struct kobj_attribute *attr,
                            const char *buf, size_t count)
{
        if (sysfs_streq(buf, "always")) {
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
                set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
        } else if (sysfs_streq(buf, "defer+madvise")) {
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
                set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
        } else if (sysfs_streq(buf, "defer")) {
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
                set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
        } else if (sysfs_streq(buf, "madvise")) {
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
                set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
        } else if (sysfs_streq(buf, "never")) {
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
        } else
                return -EINVAL;

        return count;
}
static struct kobj_attribute defrag_attr = __ATTR_RW(defrag);

static ssize_t use_zero_page_show(struct kobject *kobj,
                                  struct kobj_attribute *attr, char *buf)
{
        return single_hugepage_flag_show(kobj, attr, buf,
                                         TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
}
static ssize_t use_zero_page_store(struct kobject *kobj,
                struct kobj_attribute *attr, const char *buf, size_t count)
{
        return single_hugepage_flag_store(kobj, attr, buf, count,
                                 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
}
static struct kobj_attribute use_zero_page_attr = __ATTR_RW(use_zero_page);

static ssize_t hpage_pmd_size_show(struct kobject *kobj,
                                   struct kobj_attribute *attr, char *buf)
{
        return sysfs_emit(buf, "%lu\n", HPAGE_PMD_SIZE);
}
static struct kobj_attribute hpage_pmd_size_attr =
        __ATTR_RO(hpage_pmd_size);

static ssize_t split_underused_thp_show(struct kobject *kobj,
                            struct kobj_attribute *attr, char *buf)
{
        return sysfs_emit(buf, "%d\n", split_underused_thp);
}

static ssize_t split_underused_thp_store(struct kobject *kobj,
                             struct kobj_attribute *attr,
                             const char *buf, size_t count)
{
        int err = kstrtobool(buf, &split_underused_thp);

        if (err < 0)
                return err;

        return count;
}

static struct kobj_attribute split_underused_thp_attr = __ATTR(
        shrink_underused, 0644, split_underused_thp_show, split_underused_thp_store);

static struct attribute *hugepage_attr[] = {
        &enabled_attr.attr,
        &defrag_attr.attr,
        &use_zero_page_attr.attr,
        &hpage_pmd_size_attr.attr,
#ifdef CONFIG_SHMEM
        &shmem_enabled_attr.attr,
#endif
        &split_underused_thp_attr.attr,
        NULL,
};

static const struct attribute_group hugepage_attr_group = {
        .attrs = hugepage_attr,
};

static void hugepage_exit_sysfs(struct kobject *hugepage_kobj);
static void thpsize_release(struct kobject *kobj);
static DEFINE_SPINLOCK(huge_anon_orders_lock);
static LIST_HEAD(thpsize_list);

static ssize_t anon_enabled_show(struct kobject *kobj,
                                 struct kobj_attribute *attr, char *buf)
{
        int order = to_thpsize(kobj)->order;
        const char *output;

        if (test_bit(order, &huge_anon_orders_always))
                output = "[always] inherit madvise never";
        else if (test_bit(order, &huge_anon_orders_inherit))
                output = "always [inherit] madvise never";
        else if (test_bit(order, &huge_anon_orders_madvise))
                output = "always inherit [madvise] never";
        else
                output = "always inherit madvise [never]";

        return sysfs_emit(buf, "%s\n", output);
}

static ssize_t anon_enabled_store(struct kobject *kobj,
                                  struct kobj_attribute *attr,
                                  const char *buf, size_t count)
{
        int order = to_thpsize(kobj)->order;
        ssize_t ret = count;

        if (sysfs_streq(buf, "always")) {
                spin_lock(&huge_anon_orders_lock);
                clear_bit(order, &huge_anon_orders_inherit);
                clear_bit(order, &huge_anon_orders_madvise);
                set_bit(order, &huge_anon_orders_always);
                spin_unlock(&huge_anon_orders_lock);
        } else if (sysfs_streq(buf, "inherit")) {
                spin_lock(&huge_anon_orders_lock);
                clear_bit(order, &huge_anon_orders_always);
                clear_bit(order, &huge_anon_orders_madvise);
                set_bit(order, &huge_anon_orders_inherit);
                spin_unlock(&huge_anon_orders_lock);
        } else if (sysfs_streq(buf, "madvise")) {
                spin_lock(&huge_anon_orders_lock);
                clear_bit(order, &huge_anon_orders_always);
                clear_bit(order, &huge_anon_orders_inherit);
                set_bit(order, &huge_anon_orders_madvise);
                spin_unlock(&huge_anon_orders_lock);
        } else if (sysfs_streq(buf, "never")) {
                spin_lock(&huge_anon_orders_lock);
                clear_bit(order, &huge_anon_orders_always);
                clear_bit(order, &huge_anon_orders_inherit);
                clear_bit(order, &huge_anon_orders_madvise);
                spin_unlock(&huge_anon_orders_lock);
        } else
                ret = -EINVAL;

        if (ret > 0) {
                int err;

                err = start_stop_khugepaged();
                if (err)
                        ret = err;
        }
        return ret;
}

static struct kobj_attribute anon_enabled_attr =
        __ATTR(enabled, 0644, anon_enabled_show, anon_enabled_store);

static struct attribute *anon_ctrl_attrs[] = {
        &anon_enabled_attr.attr,
        NULL,
};

static const struct attribute_group anon_ctrl_attr_grp = {
        .attrs = anon_ctrl_attrs,
};

static struct attribute *file_ctrl_attrs[] = {
#ifdef CONFIG_SHMEM
        &thpsize_shmem_enabled_attr.attr,
#endif
        NULL,
};

static const struct attribute_group file_ctrl_attr_grp = {
        .attrs = file_ctrl_attrs,
};

static struct attribute *any_ctrl_attrs[] = {
        NULL,
};

static const struct attribute_group any_ctrl_attr_grp = {
        .attrs = any_ctrl_attrs,
};

static const struct kobj_type thpsize_ktype = {
        .release = &thpsize_release,
        .sysfs_ops = &kobj_sysfs_ops,
};

DEFINE_PER_CPU(struct mthp_stat, mthp_stats) = {{{0}}};

static unsigned long sum_mthp_stat(int order, enum mthp_stat_item item)
{
        unsigned long sum = 0;
        int cpu;

        for_each_possible_cpu(cpu) {
                struct mthp_stat *this = &per_cpu(mthp_stats, cpu);

                sum += this->stats[order][item];
        }

        return sum;
}

#define DEFINE_MTHP_STAT_ATTR(_name, _index)                                \
static ssize_t _name##_show(struct kobject *kobj,                        \
                        struct kobj_attribute *attr, char *buf)                \
{                                                                        \
        int order = to_thpsize(kobj)->order;                                \
                                                                        \
        return sysfs_emit(buf, "%lu\n", sum_mthp_stat(order, _index));        \
}                                                                        \
static struct kobj_attribute _name##_attr = __ATTR_RO(_name)

DEFINE_MTHP_STAT_ATTR(anon_fault_alloc, MTHP_STAT_ANON_FAULT_ALLOC);
DEFINE_MTHP_STAT_ATTR(anon_fault_fallback, MTHP_STAT_ANON_FAULT_FALLBACK);
DEFINE_MTHP_STAT_ATTR(anon_fault_fallback_charge, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE);
DEFINE_MTHP_STAT_ATTR(zswpout, MTHP_STAT_ZSWPOUT);
DEFINE_MTHP_STAT_ATTR(swpin, MTHP_STAT_SWPIN);
DEFINE_MTHP_STAT_ATTR(swpin_fallback, MTHP_STAT_SWPIN_FALLBACK);
DEFINE_MTHP_STAT_ATTR(swpin_fallback_charge, MTHP_STAT_SWPIN_FALLBACK_CHARGE);
DEFINE_MTHP_STAT_ATTR(swpout, MTHP_STAT_SWPOUT);
DEFINE_MTHP_STAT_ATTR(swpout_fallback, MTHP_STAT_SWPOUT_FALLBACK);
#ifdef CONFIG_SHMEM
DEFINE_MTHP_STAT_ATTR(shmem_alloc, MTHP_STAT_SHMEM_ALLOC);
DEFINE_MTHP_STAT_ATTR(shmem_fallback, MTHP_STAT_SHMEM_FALLBACK);
DEFINE_MTHP_STAT_ATTR(shmem_fallback_charge, MTHP_STAT_SHMEM_FALLBACK_CHARGE);
#endif
DEFINE_MTHP_STAT_ATTR(split, MTHP_STAT_SPLIT);
DEFINE_MTHP_STAT_ATTR(split_failed, MTHP_STAT_SPLIT_FAILED);
DEFINE_MTHP_STAT_ATTR(split_deferred, MTHP_STAT_SPLIT_DEFERRED);
DEFINE_MTHP_STAT_ATTR(nr_anon, MTHP_STAT_NR_ANON);
DEFINE_MTHP_STAT_ATTR(nr_anon_partially_mapped, MTHP_STAT_NR_ANON_PARTIALLY_MAPPED);

static struct attribute *anon_stats_attrs[] = {
        &anon_fault_alloc_attr.attr,
        &anon_fault_fallback_attr.attr,
        &anon_fault_fallback_charge_attr.attr,
#ifndef CONFIG_SHMEM
        &zswpout_attr.attr,
        &swpin_attr.attr,
        &swpin_fallback_attr.attr,
        &swpin_fallback_charge_attr.attr,
        &swpout_attr.attr,
        &swpout_fallback_attr.attr,
#endif
        &split_deferred_attr.attr,
        &nr_anon_attr.attr,
        &nr_anon_partially_mapped_attr.attr,
        NULL,
};

static struct attribute_group anon_stats_attr_grp = {
        .name = "stats",
        .attrs = anon_stats_attrs,
};

static struct attribute *file_stats_attrs[] = {
#ifdef CONFIG_SHMEM
        &shmem_alloc_attr.attr,
        &shmem_fallback_attr.attr,
        &shmem_fallback_charge_attr.attr,
#endif
        NULL,
};

static struct attribute_group file_stats_attr_grp = {
        .name = "stats",
        .attrs = file_stats_attrs,
};

static struct attribute *any_stats_attrs[] = {
#ifdef CONFIG_SHMEM
        &zswpout_attr.attr,
        &swpin_attr.attr,
        &swpin_fallback_attr.attr,
        &swpin_fallback_charge_attr.attr,
        &swpout_attr.attr,
        &swpout_fallback_attr.attr,
#endif
        &split_attr.attr,
        &split_failed_attr.attr,
        NULL,
};

static struct attribute_group any_stats_attr_grp = {
        .name = "stats",
        .attrs = any_stats_attrs,
};

static int sysfs_add_group(struct kobject *kobj,
                           const struct attribute_group *grp)
{
        int ret = -ENOENT;

        /*
         * If the group is named, try to merge first, assuming the subdirectory
         * was already created. This avoids the warning emitted by
         * sysfs_create_group() if the directory already exists.
         */
        if (grp->name)
                ret = sysfs_merge_group(kobj, grp);
        if (ret)
                ret = sysfs_create_group(kobj, grp);

        return ret;
}

static struct thpsize *thpsize_create(int order, struct kobject *parent)
{
        unsigned long size = (PAGE_SIZE << order) / SZ_1K;
        struct thpsize *thpsize;
        int ret = -ENOMEM;

        thpsize = kzalloc(sizeof(*thpsize), GFP_KERNEL);
        if (!thpsize)
                goto err;

        thpsize->order = order;

        ret = kobject_init_and_add(&thpsize->kobj, &thpsize_ktype, parent,
                                   "hugepages-%lukB", size);
        if (ret) {
                kfree(thpsize);
                goto err;
        }


        ret = sysfs_add_group(&thpsize->kobj, &any_ctrl_attr_grp);
        if (ret)
                goto err_put;

        ret = sysfs_add_group(&thpsize->kobj, &any_stats_attr_grp);
        if (ret)
                goto err_put;

        if (BIT(order) & THP_ORDERS_ALL_ANON) {
                ret = sysfs_add_group(&thpsize->kobj, &anon_ctrl_attr_grp);
                if (ret)
                        goto err_put;

                ret = sysfs_add_group(&thpsize->kobj, &anon_stats_attr_grp);
                if (ret)
                        goto err_put;
        }

        if (BIT(order) & THP_ORDERS_ALL_FILE_DEFAULT) {
                ret = sysfs_add_group(&thpsize->kobj, &file_ctrl_attr_grp);
                if (ret)
                        goto err_put;

                ret = sysfs_add_group(&thpsize->kobj, &file_stats_attr_grp);
                if (ret)
                        goto err_put;
        }

        return thpsize;
err_put:
        kobject_put(&thpsize->kobj);
err:
        return ERR_PTR(ret);
}

static void thpsize_release(struct kobject *kobj)
{
        kfree(to_thpsize(kobj));
}

static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj)
{
        int err;
        struct thpsize *thpsize;
        unsigned long orders;
        int order;

        /*
         * Default to setting PMD-sized THP to inherit the global setting and
         * disable all other sizes. powerpc's PMD_ORDER isn't a compile-time
         * constant so we have to do this here.
         */
        if (!anon_orders_configured)
                huge_anon_orders_inherit = BIT(PMD_ORDER);

        *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj);
        if (unlikely(!*hugepage_kobj)) {
                pr_err("failed to create transparent hugepage kobject\n");
                return -ENOMEM;
        }

        err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group);
        if (err) {
                pr_err("failed to register transparent hugepage group\n");
                goto delete_obj;
        }

        err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group);
        if (err) {
                pr_err("failed to register transparent hugepage group\n");
                goto remove_hp_group;
        }

        orders = THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_FILE_DEFAULT;
        order = highest_order(orders);
        while (orders) {
                thpsize = thpsize_create(order, *hugepage_kobj);
                if (IS_ERR(thpsize)) {
                        pr_err("failed to create thpsize for order %d\n", order);
                        err = PTR_ERR(thpsize);
                        goto remove_all;
                }
                list_add(&thpsize->node, &thpsize_list);
                order = next_order(&orders, order);
        }

        return 0;

remove_all:
        hugepage_exit_sysfs(*hugepage_kobj);
        return err;
remove_hp_group:
        sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group);
delete_obj:
        kobject_put(*hugepage_kobj);
        return err;
}

static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj)
{
        struct thpsize *thpsize, *tmp;

        list_for_each_entry_safe(thpsize, tmp, &thpsize_list, node) {
                list_del(&thpsize->node);
                kobject_put(&thpsize->kobj);
        }

        sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group);
        sysfs_remove_group(hugepage_kobj, &hugepage_attr_group);
        kobject_put(hugepage_kobj);
}
#else
static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj)
{
        return 0;
}

static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj)
{
}
#endif /* CONFIG_SYSFS */

static int __init thp_shrinker_init(void)
{
        huge_zero_page_shrinker = shrinker_alloc(0, "thp-zero");
        if (!huge_zero_page_shrinker)
                return -ENOMEM;

        deferred_split_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE |
                                                 SHRINKER_MEMCG_AWARE |
                                                 SHRINKER_NONSLAB,
                                                 "thp-deferred_split");
        if (!deferred_split_shrinker) {
                shrinker_free(huge_zero_page_shrinker);
                return -ENOMEM;
        }

        huge_zero_page_shrinker->count_objects = shrink_huge_zero_page_count;
        huge_zero_page_shrinker->scan_objects = shrink_huge_zero_page_scan;
        shrinker_register(huge_zero_page_shrinker);

        deferred_split_shrinker->count_objects = deferred_split_count;
        deferred_split_shrinker->scan_objects = deferred_split_scan;
        shrinker_register(deferred_split_shrinker);

        return 0;
}

static void __init thp_shrinker_exit(void)
{
        shrinker_free(huge_zero_page_shrinker);
        shrinker_free(deferred_split_shrinker);
}

static int __init hugepage_init(void)
{
        int err;
        struct kobject *hugepage_kobj;

        if (!has_transparent_hugepage()) {
                transparent_hugepage_flags = 1 << TRANSPARENT_HUGEPAGE_UNSUPPORTED;
                return -EINVAL;
        }

        /*
         * hugepages can't be allocated by the buddy allocator
         */
        MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER > MAX_PAGE_ORDER);

        err = hugepage_init_sysfs(&hugepage_kobj);
        if (err)
                goto err_sysfs;

        err = khugepaged_init();
        if (err)
                goto err_slab;

        err = thp_shrinker_init();
        if (err)
                goto err_shrinker;

        /*
         * By default disable transparent hugepages on smaller systems,
         * where the extra memory used could hurt more than TLB overhead
         * is likely to save.  The admin can still enable it through /sys.
         */
        if (totalram_pages() < (512 << (20 - PAGE_SHIFT))) {
                transparent_hugepage_flags = 0;
                return 0;
        }

        err = start_stop_khugepaged();
        if (err)
                goto err_khugepaged;

        return 0;
err_khugepaged:
        thp_shrinker_exit();
err_shrinker:
        khugepaged_destroy();
err_slab:
        hugepage_exit_sysfs(hugepage_kobj);
err_sysfs:
        return err;
}
subsys_initcall(hugepage_init);

static int __init setup_transparent_hugepage(char *str)
{
        int ret = 0;
        if (!str)
                goto out;
        if (!strcmp(str, "always")) {
                set_bit(TRANSPARENT_HUGEPAGE_FLAG,
                        &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
                          &transparent_hugepage_flags);
                ret = 1;
        } else if (!strcmp(str, "madvise")) {
                clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
                          &transparent_hugepage_flags);
                set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
                        &transparent_hugepage_flags);
                ret = 1;
        } else if (!strcmp(str, "never")) {
                clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
                          &transparent_hugepage_flags);
                clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
                          &transparent_hugepage_flags);
                ret = 1;
        }
out:
        if (!ret)
                pr_warn("transparent_hugepage= cannot parse, ignored\n");
        return ret;
}
__setup("transparent_hugepage=", setup_transparent_hugepage);

static char str_dup[PAGE_SIZE] __initdata;
static int __init setup_thp_anon(char *str)
{
        char *token, *range, *policy, *subtoken;
        unsigned long always, inherit, madvise;
        char *start_size, *end_size;
        int start, end, nr;
        char *p;

        if (!str || strlen(str) + 1 > PAGE_SIZE)
                goto err;
        strscpy(str_dup, str);

        always = huge_anon_orders_always;
        madvise = huge_anon_orders_madvise;
        inherit = huge_anon_orders_inherit;
        p = str_dup;
        while ((token = strsep(&p, ";")) != NULL) {
                range = strsep(&token, ":");
                policy = token;

                if (!policy)
                        goto err;

                while ((subtoken = strsep(&range, ",")) != NULL) {
                        if (strchr(subtoken, '-')) {
                                start_size = strsep(&subtoken, "-");
                                end_size = subtoken;

                                start = get_order_from_str(start_size, THP_ORDERS_ALL_ANON);
                                end = get_order_from_str(end_size, THP_ORDERS_ALL_ANON);
                        } else {
                                start_size = end_size = subtoken;
                                start = end = get_order_from_str(subtoken,
                                                                 THP_ORDERS_ALL_ANON);
                        }

                        if (start == -EINVAL) {
                                pr_err("invalid size %s in thp_anon boot parameter\n", start_size);
                                goto err;
                        }

                        if (end == -EINVAL) {
                                pr_err("invalid size %s in thp_anon boot parameter\n", end_size);
                                goto err;
                        }

                        if (start < 0 || end < 0 || start > end)
                                goto err;

                        nr = end - start + 1;
                        if (!strcmp(policy, "always")) {
                                bitmap_set(&always, start, nr);
                                bitmap_clear(&inherit, start, nr);
                                bitmap_clear(&madvise, start, nr);
                        } else if (!strcmp(policy, "madvise")) {
                                bitmap_set(&madvise, start, nr);
                                bitmap_clear(&inherit, start, nr);
                                bitmap_clear(&always, start, nr);
                        } else if (!strcmp(policy, "inherit")) {
                                bitmap_set(&inherit, start, nr);
                                bitmap_clear(&madvise, start, nr);
                                bitmap_clear(&always, start, nr);
                        } else if (!strcmp(policy, "never")) {
                                bitmap_clear(&inherit, start, nr);
                                bitmap_clear(&madvise, start, nr);
                                bitmap_clear(&always, start, nr);
                        } else {
                                pr_err("invalid policy %s in thp_anon boot parameter\n", policy);
                                goto err;
                        }
                }
        }

        huge_anon_orders_always = always;
        huge_anon_orders_madvise = madvise;
        huge_anon_orders_inherit = inherit;
        anon_orders_configured = true;
        return 1;

err:
        pr_warn("thp_anon=%s: error parsing string, ignoring setting\n", str);
        return 0;
}
__setup("thp_anon=", setup_thp_anon);

pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
{
        if (likely(vma->vm_flags & VM_WRITE))
                pmd = pmd_mkwrite(pmd, vma);
        return pmd;
}

#ifdef CONFIG_MEMCG
static inline
struct deferred_split *get_deferred_split_queue(struct folio *folio)
{
        struct mem_cgroup *memcg = folio_memcg(folio);
        struct pglist_data *pgdat = NODE_DATA(folio_nid(folio));

        if (memcg)
                return &memcg->deferred_split_queue;
        else
                return &pgdat->deferred_split_queue;
}
#else
static inline
struct deferred_split *get_deferred_split_queue(struct folio *folio)
{
        struct pglist_data *pgdat = NODE_DATA(folio_nid(folio));

        return &pgdat->deferred_split_queue;
}
#endif

static inline bool is_transparent_hugepage(const struct folio *folio)
{
        if (!folio_test_large(folio))
                return false;

        return is_huge_zero_folio(folio) ||
                folio_test_large_rmappable(folio);
}

static unsigned long __thp_get_unmapped_area(struct file *filp,
                unsigned long addr, unsigned long len,
                loff_t off, unsigned long flags, unsigned long size,
                vm_flags_t vm_flags)
{
        loff_t off_end = off + len;
        loff_t off_align = round_up(off, size);
        unsigned long len_pad, ret, off_sub;

        if (!IS_ENABLED(CONFIG_64BIT) || in_compat_syscall())
                return 0;

        if (off_end <= off_align || (off_end - off_align) < size)
                return 0;

        len_pad = len + size;
        if (len_pad < len || (off + len_pad) < off)
                return 0;

        ret = mm_get_unmapped_area_vmflags(current->mm, filp, addr, len_pad,
                                           off >> PAGE_SHIFT, flags, vm_flags);

        /*
         * The failure might be due to length padding. The caller will retry
         * without the padding.
         */
        if (IS_ERR_VALUE(ret))
                return 0;

        /*
         * Do not try to align to THP boundary if allocation at the address
         * hint succeeds.
         */
        if (ret == addr)
                return addr;

        off_sub = (off - ret) & (size - 1);

        if (test_bit(MMF_TOPDOWN, &current->mm->flags) && !off_sub)
                return ret + size;

        ret += off_sub;
        return ret;
}

unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr,
                unsigned long len, unsigned long pgoff, unsigned long flags,
                vm_flags_t vm_flags)
{
        unsigned long ret;
        loff_t off = (loff_t)pgoff << PAGE_SHIFT;

        ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, vm_flags);
        if (ret)
                return ret;

        return mm_get_unmapped_area_vmflags(current->mm, filp, addr, len, pgoff, flags,
                                            vm_flags);
}

unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
                unsigned long len, unsigned long pgoff, unsigned long flags)
{
        return thp_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, 0);
}
EXPORT_SYMBOL_GPL(thp_get_unmapped_area);

static struct folio *vma_alloc_anon_folio_pmd(struct vm_area_struct *vma,
                unsigned long addr)
{
        gfp_t gfp = vma_thp_gfp_mask(vma);
        const int order = HPAGE_PMD_ORDER;
        struct folio *folio;

        folio = vma_alloc_folio(gfp, order, vma, addr & HPAGE_PMD_MASK);

        if (unlikely(!folio)) {
                count_vm_event(THP_FAULT_FALLBACK);
                count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK);
                return NULL;
        }

        VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
        if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) {
                folio_put(folio);
                count_vm_event(THP_FAULT_FALLBACK);
                count_vm_event(THP_FAULT_FALLBACK_CHARGE);
                count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK);
                count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE);
                return NULL;
        }
        folio_throttle_swaprate(folio, gfp);

       /*
        * When a folio is not zeroed during allocation (__GFP_ZERO not used)
        * or user folios require special handling, folio_zero_user() is used to
        * make sure that the page corresponding to the faulting address will be
        * hot in the cache after zeroing.
        */
        if (user_alloc_needs_zeroing())
                folio_zero_user(folio, addr);
        /*
         * The memory barrier inside __folio_mark_uptodate makes sure that
         * folio_zero_user writes become visible before the set_pmd_at()
         * write.
         */
        __folio_mark_uptodate(folio);
        return folio;
}

static void map_anon_folio_pmd(struct folio *folio, pmd_t *pmd,
                struct vm_area_struct *vma, unsigned long haddr)
{
        pmd_t entry;

        entry = mk_huge_pmd(&folio->page, vma->vm_page_prot);
        entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
        folio_add_new_anon_rmap(folio, vma, haddr, RMAP_EXCLUSIVE);
        folio_add_lru_vma(folio, vma);
        set_pmd_at(vma->vm_mm, haddr, pmd, entry);
        update_mmu_cache_pmd(vma, haddr, pmd);
        add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
        count_vm_event(THP_FAULT_ALLOC);
        count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_ALLOC);
        count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC);
}

static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf)
{
        unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
        struct vm_area_struct *vma = vmf->vma;
        struct folio *folio;
        pgtable_t pgtable;
        vm_fault_t ret = 0;

        folio = vma_alloc_anon_folio_pmd(vma, vmf->address);
        if (unlikely(!folio))
                return VM_FAULT_FALLBACK;

        pgtable = pte_alloc_one(vma->vm_mm);
        if (unlikely(!pgtable)) {
                ret = VM_FAULT_OOM;
                goto release;
        }

        vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
        if (unlikely(!pmd_none(*vmf->pmd))) {
                goto unlock_release;
        } else {
                ret = check_stable_address_space(vma->vm_mm);
                if (ret)
                        goto unlock_release;

                /* Deliver the page fault to userland */
                if (userfaultfd_missing(vma)) {
                        spin_unlock(vmf->ptl);
                        folio_put(folio);
                        pte_free(vma->vm_mm, pgtable);
                        ret = handle_userfault(vmf, VM_UFFD_MISSING);
                        VM_BUG_ON(ret & VM_FAULT_FALLBACK);
                        return ret;
                }
                pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);
                map_anon_folio_pmd(folio, vmf->pmd, vma, haddr);
                mm_inc_nr_ptes(vma->vm_mm);
                deferred_split_folio(folio, false);
                spin_unlock(vmf->ptl);
        }

        return 0;
unlock_release:
        spin_unlock(vmf->ptl);
release:
        if (pgtable)
                pte_free(vma->vm_mm, pgtable);
        folio_put(folio);
        return ret;

}

/*
 * always: directly stall for all thp allocations
 * defer: wake kswapd and fail if not immediately available
 * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise
 *                  fail if not immediately available
 * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately
 *            available
 * never: never stall for any thp allocation
 */
gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma)
{
        const bool vma_madvised = vma && (vma->vm_flags & VM_HUGEPAGE);

        /* Always do synchronous compaction */
        if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
                return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY);

        /* Kick kcompactd and fail quickly */
        if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
                return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM;

        /* Synchronous compaction if madvised, otherwise kick kcompactd */
        if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags))
                return GFP_TRANSHUGE_LIGHT |
                        (vma_madvised ? __GFP_DIRECT_RECLAIM :
                                        __GFP_KSWAPD_RECLAIM);

        /* Only do synchronous compaction if madvised */
        if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
                return GFP_TRANSHUGE_LIGHT |
                       (vma_madvised ? __GFP_DIRECT_RECLAIM : 0);

        return GFP_TRANSHUGE_LIGHT;
}

/* Caller must hold page table lock. */
static void set_huge_zero_folio(pgtable_t pgtable, struct mm_struct *mm,
                struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,
                struct folio *zero_folio)
{
        pmd_t entry;
        entry = mk_pmd(&zero_folio->page, vma->vm_page_prot);
        entry = pmd_mkhuge(entry);
        pgtable_trans_huge_deposit(mm, pmd, pgtable);
        set_pmd_at(mm, haddr, pmd, entry);
        mm_inc_nr_ptes(mm);
}

vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
        vm_fault_t ret;

        if (!thp_vma_suitable_order(vma, haddr, PMD_ORDER))
                return VM_FAULT_FALLBACK;
        ret = vmf_anon_prepare(vmf);
        if (ret)
                return ret;
        khugepaged_enter_vma(vma, vma->vm_flags);

        if (!(vmf->flags & FAULT_FLAG_WRITE) &&
                        !mm_forbids_zeropage(vma->vm_mm) &&
                        transparent_hugepage_use_zero_page()) {
                pgtable_t pgtable;
                struct folio *zero_folio;
                vm_fault_t ret;

                pgtable = pte_alloc_one(vma->vm_mm);
                if (unlikely(!pgtable))
                        return VM_FAULT_OOM;
                zero_folio = mm_get_huge_zero_folio(vma->vm_mm);
                if (unlikely(!zero_folio)) {
                        pte_free(vma->vm_mm, pgtable);
                        count_vm_event(THP_FAULT_FALLBACK);
                        return VM_FAULT_FALLBACK;
                }
                vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
                ret = 0;
                if (pmd_none(*vmf->pmd)) {
                        ret = check_stable_address_space(vma->vm_mm);
                        if (ret) {
                                spin_unlock(vmf->ptl);
                                pte_free(vma->vm_mm, pgtable);
                        } else if (userfaultfd_missing(vma)) {
                                spin_unlock(vmf->ptl);
                                pte_free(vma->vm_mm, pgtable);
                                ret = handle_userfault(vmf, VM_UFFD_MISSING);
                                VM_BUG_ON(ret & VM_FAULT_FALLBACK);
                        } else {
                                set_huge_zero_folio(pgtable, vma->vm_mm, vma,
                                                   haddr, vmf->pmd, zero_folio);
                                update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
                                spin_unlock(vmf->ptl);
                        }
                } else {
                        spin_unlock(vmf->ptl);
                        pte_free(vma->vm_mm, pgtable);
                }
                return ret;
        }

        return __do_huge_pmd_anonymous_page(vmf);
}

static int insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
                pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write,
                pgtable_t pgtable)
{
        struct mm_struct *mm = vma->vm_mm;
        pmd_t entry;

        lockdep_assert_held(pmd_lockptr(mm, pmd));

        if (!pmd_none(*pmd)) {
                if (write) {
                        if (pmd_pfn(*pmd) != pfn_t_to_pfn(pfn)) {
                                WARN_ON_ONCE(!is_huge_zero_pmd(*pmd));
                                return -EEXIST;
                        }
                        entry = pmd_mkyoung(*pmd);
                        entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
                        if (pmdp_set_access_flags(vma, addr, pmd, entry, 1))
                                update_mmu_cache_pmd(vma, addr, pmd);
                }

                return -EEXIST;
        }

        entry = pmd_mkhuge(pfn_t_pmd(pfn, prot));
        if (pfn_t_devmap(pfn))
                entry = pmd_mkdevmap(entry);
        else
                entry = pmd_mkspecial(entry);
        if (write) {
                entry = pmd_mkyoung(pmd_mkdirty(entry));
                entry = maybe_pmd_mkwrite(entry, vma);
        }

        if (pgtable) {
                pgtable_trans_huge_deposit(mm, pmd, pgtable);
                mm_inc_nr_ptes(mm);
        }

        set_pmd_at(mm, addr, pmd, entry);
        update_mmu_cache_pmd(vma, addr, pmd);
        return 0;
}

/**
 * vmf_insert_pfn_pmd - insert a pmd size pfn
 * @vmf: Structure describing the fault
 * @pfn: pfn to insert
 * @write: whether it's a write fault
 *
 * Insert a pmd size pfn. See vmf_insert_pfn() for additional info.
 *
 * Return: vm_fault_t value.
 */
vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write)
{
        unsigned long addr = vmf->address & PMD_MASK;
        struct vm_area_struct *vma = vmf->vma;
        pgprot_t pgprot = vma->vm_page_prot;
        pgtable_t pgtable = NULL;
        spinlock_t *ptl;
        int error;

        /*
         * If we had pmd_special, we could avoid all these restrictions,
         * but we need to be consistent with PTEs and architectures that
         * can't support a 'special' bit.
         */
        BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) &&
                        !pfn_t_devmap(pfn));
        BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
                                                (VM_PFNMAP|VM_MIXEDMAP));
        BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));

        if (addr < vma->vm_start || addr >= vma->vm_end)
                return VM_FAULT_SIGBUS;

        if (arch_needs_pgtable_deposit()) {
                pgtable = pte_alloc_one(vma->vm_mm);
                if (!pgtable)
                        return VM_FAULT_OOM;
        }

        track_pfn_insert(vma, &pgprot, pfn);
        ptl = pmd_lock(vma->vm_mm, vmf->pmd);
        error = insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write,
                        pgtable);
        spin_unlock(ptl);
        if (error && pgtable)
                pte_free(vma->vm_mm, pgtable);

        return VM_FAULT_NOPAGE;
}
EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd);

vm_fault_t vmf_insert_folio_pmd(struct vm_fault *vmf, struct folio *folio,
                                bool write)
{
        struct vm_area_struct *vma = vmf->vma;
        unsigned long addr = vmf->address & PMD_MASK;
        struct mm_struct *mm = vma->vm_mm;
        spinlock_t *ptl;
        pgtable_t pgtable = NULL;
        int error;

        if (addr < vma->vm_start || addr >= vma->vm_end)
                return VM_FAULT_SIGBUS;

        if (WARN_ON_ONCE(folio_order(folio) != PMD_ORDER))
                return VM_FAULT_SIGBUS;

        if (arch_needs_pgtable_deposit()) {
                pgtable = pte_alloc_one(vma->vm_mm);
                if (!pgtable)
                        return VM_FAULT_OOM;
        }

        ptl = pmd_lock(mm, vmf->pmd);
        if (pmd_none(*vmf->pmd)) {
                folio_get(folio);
                folio_add_file_rmap_pmd(folio, &folio->page, vma);
                add_mm_counter(mm, mm_counter_file(folio), HPAGE_PMD_NR);
        }
        error = insert_pfn_pmd(vma, addr, vmf->pmd,
                        pfn_to_pfn_t(folio_pfn(folio)), vma->vm_page_prot,
                        write, pgtable);
        spin_unlock(ptl);
        if (error && pgtable)
                pte_free(mm, pgtable);

        return VM_FAULT_NOPAGE;
}
EXPORT_SYMBOL_GPL(vmf_insert_folio_pmd);

#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma)
{
        if (likely(vma->vm_flags & VM_WRITE))
                pud = pud_mkwrite(pud);
        return pud;
}

static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr,
                pud_t *pud, pfn_t pfn, bool write)
{
        struct mm_struct *mm = vma->vm_mm;
        pgprot_t prot = vma->vm_page_prot;
        pud_t entry;

        if (!pud_none(*pud)) {
                if (write) {
                        if (WARN_ON_ONCE(pud_pfn(*pud) != pfn_t_to_pfn(pfn)))
                                return;
                        entry = pud_mkyoung(*pud);
                        entry = maybe_pud_mkwrite(pud_mkdirty(entry), vma);
                        if (pudp_set_access_flags(vma, addr, pud, entry, 1))
                                update_mmu_cache_pud(vma, addr, pud);
                }
                return;
        }

        entry = pud_mkhuge(pfn_t_pud(pfn, prot));
        if (pfn_t_devmap(pfn))
                entry = pud_mkdevmap(entry);
        else
                entry = pud_mkspecial(entry);
        if (write) {
                entry = pud_mkyoung(pud_mkdirty(entry));
                entry = maybe_pud_mkwrite(entry, vma);
        }
        set_pud_at(mm, addr, pud, entry);
        update_mmu_cache_pud(vma, addr, pud);
}

/**
 * vmf_insert_pfn_pud - insert a pud size pfn
 * @vmf: Structure describing the fault
 * @pfn: pfn to insert
 * @write: whether it's a write fault
 *
 * Insert a pud size pfn. See vmf_insert_pfn() for additional info.
 *
 * Return: vm_fault_t value.
 */
vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write)
{
        unsigned long addr = vmf->address & PUD_MASK;
        struct vm_area_struct *vma = vmf->vma;
        pgprot_t pgprot = vma->vm_page_prot;
        spinlock_t *ptl;

        /*
         * If we had pud_special, we could avoid all these restrictions,
         * but we need to be consistent with PTEs and architectures that
         * can't support a 'special' bit.
         */
        BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) &&
                        !pfn_t_devmap(pfn));
        BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
                                                (VM_PFNMAP|VM_MIXEDMAP));
        BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));

        if (addr < vma->vm_start || addr >= vma->vm_end)
                return VM_FAULT_SIGBUS;

        track_pfn_insert(vma, &pgprot, pfn);

        ptl = pud_lock(vma->vm_mm, vmf->pud);
        insert_pfn_pud(vma, addr, vmf->pud, pfn, write);
        spin_unlock(ptl);

        return VM_FAULT_NOPAGE;
}
EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud);

/**
 * vmf_insert_folio_pud - insert a pud size folio mapped by a pud entry
 * @vmf: Structure describing the fault
 * @folio: folio to insert
 * @write: whether it's a write fault
 *
 * Return: vm_fault_t value.
 */
vm_fault_t vmf_insert_folio_pud(struct vm_fault *vmf, struct folio *folio,
                                bool write)
{
        struct vm_area_struct *vma = vmf->vma;
        unsigned long addr = vmf->address & PUD_MASK;
        pud_t *pud = vmf->pud;
        struct mm_struct *mm = vma->vm_mm;
        spinlock_t *ptl;

        if (addr < vma->vm_start || addr >= vma->vm_end)
                return VM_FAULT_SIGBUS;

        if (WARN_ON_ONCE(folio_order(folio) != PUD_ORDER))
                return VM_FAULT_SIGBUS;

        ptl = pud_lock(mm, pud);

        /*
         * If there is already an entry present we assume the folio is
         * already mapped, hence no need to take another reference. We
         * still call insert_pfn_pud() though in case the mapping needs
         * upgrading to writeable.
         */
        if (pud_none(*vmf->pud)) {
                folio_get(folio);
                folio_add_file_rmap_pud(folio, &folio->page, vma);
                add_mm_counter(mm, mm_counter_file(folio), HPAGE_PUD_NR);
        }
        insert_pfn_pud(vma, addr, vmf->pud, pfn_to_pfn_t(folio_pfn(folio)),
                write);
        spin_unlock(ptl);

        return VM_FAULT_NOPAGE;
}
EXPORT_SYMBOL_GPL(vmf_insert_folio_pud);
#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */

void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
               pmd_t *pmd, bool write)
{
        pmd_t _pmd;

        _pmd = pmd_mkyoung(*pmd);
        if (write)
                _pmd = pmd_mkdirty(_pmd);
        if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK,
                                  pmd, _pmd, write))
                update_mmu_cache_pmd(vma, addr, pmd);
}

struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
                pmd_t *pmd, int flags, struct dev_pagemap **pgmap)
{
        unsigned long pfn = pmd_pfn(*pmd);
        struct mm_struct *mm = vma->vm_mm;
        struct page *page;
        int ret;

        assert_spin_locked(pmd_lockptr(mm, pmd));

        if (flags & FOLL_WRITE && !pmd_write(*pmd))
                return NULL;

        if (pmd_present(*pmd) && pmd_devmap(*pmd))
                /* pass */;
        else
                return NULL;

        if (flags & FOLL_TOUCH)
                touch_pmd(vma, addr, pmd, flags & FOLL_WRITE);

        /*
         * device mapped pages can only be returned if the
         * caller will manage the page reference count.
         */
        if (!(flags & (FOLL_GET | FOLL_PIN)))
                return ERR_PTR(-EEXIST);

        pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT;
        *pgmap = get_dev_pagemap(pfn, *pgmap);
        if (!*pgmap)
                return ERR_PTR(-EFAULT);
        page = pfn_to_page(pfn);
        ret = try_grab_folio(page_folio(page), 1, flags);
        if (ret)
                page = ERR_PTR(ret);

        return page;
}

int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
                  pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
                  struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
{
        spinlock_t *dst_ptl, *src_ptl;
        struct page *src_page;
        struct folio *src_folio;
        pmd_t pmd;
        pgtable_t pgtable = NULL;
        int ret = -ENOMEM;

        pmd = pmdp_get_lockless(src_pmd);
        if (unlikely(pmd_present(pmd) && pmd_special(pmd))) {
                dst_ptl = pmd_lock(dst_mm, dst_pmd);
                src_ptl = pmd_lockptr(src_mm, src_pmd);
                spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
                /*
                 * No need to recheck the pmd, it can't change with write
                 * mmap lock held here.
                 *
                 * Meanwhile, making sure it's not a CoW VMA with writable
                 * mapping, otherwise it means either the anon page wrongly
                 * applied special bit, or we made the PRIVATE mapping be
                 * able to wrongly write to the backend MMIO.
                 */
                VM_WARN_ON_ONCE(is_cow_mapping(src_vma->vm_flags) && pmd_write(pmd));
                goto set_pmd;
        }

        /* Skip if can be re-fill on fault */
        if (!vma_is_anonymous(dst_vma))
                return 0;

        pgtable = pte_alloc_one(dst_mm);
        if (unlikely(!pgtable))
                goto out;

        dst_ptl = pmd_lock(dst_mm, dst_pmd);
        src_ptl = pmd_lockptr(src_mm, src_pmd);
        spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);

        ret = -EAGAIN;
        pmd = *src_pmd;

#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
        if (unlikely(is_swap_pmd(pmd))) {
                swp_entry_t entry = pmd_to_swp_entry(pmd);

                VM_BUG_ON(!is_pmd_migration_entry(pmd));
                if (!is_readable_migration_entry(entry)) {
                        entry = make_readable_migration_entry(
                                                        swp_offset(entry));
                        pmd = swp_entry_to_pmd(entry);
                        if (pmd_swp_soft_dirty(*src_pmd))
                                pmd = pmd_swp_mksoft_dirty(pmd);
                        if (pmd_swp_uffd_wp(*src_pmd))
                                pmd = pmd_swp_mkuffd_wp(pmd);
                        set_pmd_at(src_mm, addr, src_pmd, pmd);
                }
                add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
                mm_inc_nr_ptes(dst_mm);
                pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
                if (!userfaultfd_wp(dst_vma))
                        pmd = pmd_swp_clear_uffd_wp(pmd);
                set_pmd_at(dst_mm, addr, dst_pmd, pmd);
                ret = 0;
                goto out_unlock;
        }
#endif

        if (unlikely(!pmd_trans_huge(pmd))) {
                pte_free(dst_mm, pgtable);
                goto out_unlock;
        }
        /*
         * When page table lock is held, the huge zero pmd should not be
         * under splitting since we don't split the page itself, only pmd to
         * a page table.
         */
        if (is_huge_zero_pmd(pmd)) {
                /*
                 * mm_get_huge_zero_folio() will never allocate a new
                 * folio here, since we already have a zero page to
                 * copy. It just takes a reference.
                 */
                mm_get_huge_zero_folio(dst_mm);
                goto out_zero_page;
        }

        src_page = pmd_page(pmd);
        VM_BUG_ON_PAGE(!PageHead(src_page), src_page);
        src_folio = page_folio(src_page);

        folio_get(src_folio);
        if (unlikely(folio_try_dup_anon_rmap_pmd(src_folio, src_page, dst_vma, src_vma))) {
                /* Page maybe pinned: split and retry the fault on PTEs. */
                folio_put(src_folio);
                pte_free(dst_mm, pgtable);
                spin_unlock(src_ptl);
                spin_unlock(dst_ptl);
                __split_huge_pmd(src_vma, src_pmd, addr, false, NULL);
                return -EAGAIN;
        }
        add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
out_zero_page:
        mm_inc_nr_ptes(dst_mm);
        pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
        pmdp_set_wrprotect(src_mm, addr, src_pmd);
        if (!userfaultfd_wp(dst_vma))
                pmd = pmd_clear_uffd_wp(pmd);
        pmd = pmd_wrprotect(pmd);
set_pmd:
        pmd = pmd_mkold(pmd);
        set_pmd_at(dst_mm, addr, dst_pmd, pmd);

        ret = 0;
out_unlock:
        spin_unlock(src_ptl);
        spin_unlock(dst_ptl);
out:
        return ret;
}

#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
void touch_pud(struct vm_area_struct *vma, unsigned long addr,
               pud_t *pud, bool write)
{
        pud_t _pud;

        _pud = pud_mkyoung(*pud);
        if (write)
                _pud = pud_mkdirty(_pud);
        if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK,
                                  pud, _pud, write))
                update_mmu_cache_pud(vma, addr, pud);
}

int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm,
                  pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
                  struct vm_area_struct *vma)
{
        spinlock_t *dst_ptl, *src_ptl;
        pud_t pud;
        int ret;

        dst_ptl = pud_lock(dst_mm, dst_pud);
        src_ptl = pud_lockptr(src_mm, src_pud);
        spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);

        ret = -EAGAIN;
        pud = *src_pud;
        if (unlikely(!pud_trans_huge(pud) && !pud_devmap(pud)))
                goto out_unlock;

        /*
         * TODO: once we support anonymous pages, use
         * folio_try_dup_anon_rmap_*() and split if duplicating fails.
         */
        if (is_cow_mapping(vma->vm_flags) && pud_write(pud)) {
                pudp_set_wrprotect(src_mm, addr, src_pud);
                pud = pud_wrprotect(pud);
        }
        pud = pud_mkold(pud);
        set_pud_at(dst_mm, addr, dst_pud, pud);

        ret = 0;
out_unlock:
        spin_unlock(src_ptl);
        spin_unlock(dst_ptl);
        return ret;
}

void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)
{
        bool write = vmf->flags & FAULT_FLAG_WRITE;

        vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud);
        if (unlikely(!pud_same(*vmf->pud, orig_pud)))
                goto unlock;

        touch_pud(vmf->vma, vmf->address, vmf->pud, write);
unlock:
        spin_unlock(vmf->ptl);
}
#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */

void huge_pmd_set_accessed(struct vm_fault *vmf)
{
        bool write = vmf->flags & FAULT_FLAG_WRITE;

        vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);
        if (unlikely(!pmd_same(*vmf->pmd, vmf->orig_pmd)))
                goto unlock;

        touch_pmd(vmf->vma, vmf->address, vmf->pmd, write);

unlock:
        spin_unlock(vmf->ptl);
}

static vm_fault_t do_huge_zero_wp_pmd(struct vm_fault *vmf)
{
        unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
        struct vm_area_struct *vma = vmf->vma;
        struct mmu_notifier_range range;
        struct folio *folio;
        vm_fault_t ret = 0;

        folio = vma_alloc_anon_folio_pmd(vma, vmf->address);
        if (unlikely(!folio))
                return VM_FAULT_FALLBACK;

        mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, haddr,
                                haddr + HPAGE_PMD_SIZE);
        mmu_notifier_invalidate_range_start(&range);
        vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
        if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd)))
                goto release;
        ret = check_stable_address_space(vma->vm_mm);
        if (ret)
                goto release;
        (void)pmdp_huge_clear_flush(vma, haddr, vmf->pmd);
        map_anon_folio_pmd(folio, vmf->pmd, vma, haddr);
        goto unlock;
release:
        folio_put(folio);
unlock:
        spin_unlock(vmf->ptl);
        mmu_notifier_invalidate_range_end(&range);
        return ret;
}

vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf)
{
        const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
        struct vm_area_struct *vma = vmf->vma;
        struct folio *folio;
        struct page *page;
        unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
        pmd_t orig_pmd = vmf->orig_pmd;

        vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);
        VM_BUG_ON_VMA(!vma->anon_vma, vma);

        if (is_huge_zero_pmd(orig_pmd)) {
                vm_fault_t ret = do_huge_zero_wp_pmd(vmf);

                if (!(ret & VM_FAULT_FALLBACK))
                        return ret;

                /* Fallback to splitting PMD if THP cannot be allocated */
                goto fallback;
        }

        spin_lock(vmf->ptl);

        if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {
                spin_unlock(vmf->ptl);
                return 0;
        }

        page = pmd_page(orig_pmd);
        folio = page_folio(page);
        VM_BUG_ON_PAGE(!PageHead(page), page);

        /* Early check when only holding the PT lock. */
        if (PageAnonExclusive(page))
                goto reuse;

        if (!folio_trylock(folio)) {
                folio_get(folio);
                spin_unlock(vmf->ptl);
                folio_lock(folio);
                spin_lock(vmf->ptl);
                if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {
                        spin_unlock(vmf->ptl);
                        folio_unlock(folio);
                        folio_put(folio);
                        return 0;
                }
                folio_put(folio);
        }

        /* Recheck after temporarily dropping the PT lock. */
        if (PageAnonExclusive(page)) {
                folio_unlock(folio);
                goto reuse;
        }

        /*
         * See do_wp_page(): we can only reuse the folio exclusively if
         * there are no additional references. Note that we always drain
         * the LRU cache immediately after adding a THP.
         */
        if (folio_ref_count(folio) >
                        1 + folio_test_swapcache(folio) * folio_nr_pages(folio))
                goto unlock_fallback;
        if (folio_test_swapcache(folio))
                folio_free_swap(folio);
        if (folio_ref_count(folio) == 1) {
                pmd_t entry;

                folio_move_anon_rmap(folio, vma);
                SetPageAnonExclusive(page);
                folio_unlock(folio);
reuse:
                if (unlikely(unshare)) {
                        spin_unlock(vmf->ptl);
                        return 0;
                }
                entry = pmd_mkyoung(orig_pmd);
                entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
                if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1))
                        update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
                spin_unlock(vmf->ptl);
                return 0;
        }

unlock_fallback:
        folio_unlock(folio);
        spin_unlock(vmf->ptl);
fallback:
        __split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL);
        return VM_FAULT_FALLBACK;
}

static inline bool can_change_pmd_writable(struct vm_area_struct *vma,
                                           unsigned long addr, pmd_t pmd)
{
        struct page *page;

        if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE)))
                return false;

        /* Don't touch entries that are not even readable (NUMA hinting). */
        if (pmd_protnone(pmd))
                return false;

        /* Do we need write faults for softdirty tracking? */
        if (pmd_needs_soft_dirty_wp(vma, pmd))
                return false;

        /* Do we need write faults for uffd-wp tracking? */
        if (userfaultfd_huge_pmd_wp(vma, pmd))
                return false;

        if (!(vma->vm_flags & VM_SHARED)) {
                /* See can_change_pte_writable(). */
                page = vm_normal_page_pmd(vma, addr, pmd);
                return page && PageAnon(page) && PageAnonExclusive(page);
        }

        /* See can_change_pte_writable(). */
        return pmd_dirty(pmd);
}

/* NUMA hinting page fault entry point for trans huge pmds */
vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        struct folio *folio;
        unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
        int nid = NUMA_NO_NODE;
        int target_nid, last_cpupid;
        pmd_t pmd, old_pmd;
        bool writable = false;
        int flags = 0;

        vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
        old_pmd = pmdp_get(vmf->pmd);

        if (unlikely(!pmd_same(old_pmd, vmf->orig_pmd))) {
                spin_unlock(vmf->ptl);
                return 0;
        }

        pmd = pmd_modify(old_pmd, vma->vm_page_prot);

        /*
         * Detect now whether the PMD could be writable; this information
         * is only valid while holding the PT lock.
         */
        writable = pmd_write(pmd);
        if (!writable && vma_wants_manual_pte_write_upgrade(vma) &&
            can_change_pmd_writable(vma, vmf->address, pmd))
                writable = true;

        folio = vm_normal_folio_pmd(vma, haddr, pmd);
        if (!folio)
                goto out_map;

        nid = folio_nid(folio);

        target_nid = numa_migrate_check(folio, vmf, haddr, &flags, writable,
                                        &last_cpupid);
        if (target_nid == NUMA_NO_NODE)
                goto out_map;
        if (migrate_misplaced_folio_prepare(folio, vma, target_nid)) {
                flags |= TNF_MIGRATE_FAIL;
                goto out_map;
        }
        /* The folio is isolated and isolation code holds a folio reference. */
        spin_unlock(vmf->ptl);
        writable = false;

        if (!migrate_misplaced_folio(folio, target_nid)) {
                flags |= TNF_MIGRATED;
                nid = target_nid;
                task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags);
                return 0;
        }

        flags |= TNF_MIGRATE_FAIL;
        vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
        if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) {
                spin_unlock(vmf->ptl);
                return 0;
        }
out_map:
        /* Restore the PMD */
        pmd = pmd_modify(pmdp_get(vmf->pmd), vma->vm_page_prot);
        pmd = pmd_mkyoung(pmd);
        if (writable)
                pmd = pmd_mkwrite(pmd, vma);
        set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);
        update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
        spin_unlock(vmf->ptl);

        if (nid != NUMA_NO_NODE)
                task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags);
        return 0;
}

/*
 * Return true if we do MADV_FREE successfully on entire pmd page.
 * Otherwise, return false.
 */
bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
                pmd_t *pmd, unsigned long addr, unsigned long next)
{
        spinlock_t *ptl;
        pmd_t orig_pmd;
        struct folio *folio;
        struct mm_struct *mm = tlb->mm;
        bool ret = false;

        tlb_change_page_size(tlb, HPAGE_PMD_SIZE);

        ptl = pmd_trans_huge_lock(pmd, vma);
        if (!ptl)
                goto out_unlocked;

        orig_pmd = *pmd;
        if (is_huge_zero_pmd(orig_pmd))
                goto out;

        if (unlikely(!pmd_present(orig_pmd))) {
                VM_BUG_ON(thp_migration_supported() &&
                                  !is_pmd_migration_entry(orig_pmd));
                goto out;
        }

        folio = pmd_folio(orig_pmd);
        /*
         * If other processes are mapping this folio, we couldn't discard
         * the folio unless they all do MADV_FREE so let's skip the folio.
         */
        if (folio_maybe_mapped_shared(folio))
                goto out;

        if (!folio_trylock(folio))
                goto out;

        /*
         * If user want to discard part-pages of THP, split it so MADV_FREE
         * will deactivate only them.
         */
        if (next - addr != HPAGE_PMD_SIZE) {
                folio_get(folio);
                spin_unlock(ptl);
                split_folio(folio);
                folio_unlock(folio);
                folio_put(folio);
                goto out_unlocked;
        }

        if (folio_test_dirty(folio))
                folio_clear_dirty(folio);
        folio_unlock(folio);

        if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) {
                pmdp_invalidate(vma, addr, pmd);
                orig_pmd = pmd_mkold(orig_pmd);
                orig_pmd = pmd_mkclean(orig_pmd);

                set_pmd_at(mm, addr, pmd, orig_pmd);
                tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
        }

        folio_mark_lazyfree(folio);
        ret = true;
out:
        spin_unlock(ptl);
out_unlocked:
        return ret;
}

static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)
{
        pgtable_t pgtable;

        pgtable = pgtable_trans_huge_withdraw(mm, pmd);
        pte_free(mm, pgtable);
        mm_dec_nr_ptes(mm);
}

int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
                 pmd_t *pmd, unsigned long addr)
{
        pmd_t orig_pmd;
        spinlock_t *ptl;

        tlb_change_page_size(tlb, HPAGE_PMD_SIZE);

        ptl = __pmd_trans_huge_lock(pmd, vma);
        if (!ptl)
                return 0;
        /*
         * For architectures like ppc64 we look at deposited pgtable
         * when calling pmdp_huge_get_and_clear. So do the
         * pgtable_trans_huge_withdraw after finishing pmdp related
         * operations.
         */
        orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd,
                                                tlb->fullmm);
        arch_check_zapped_pmd(vma, orig_pmd);
        tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
        if (!vma_is_dax(vma) && vma_is_special_huge(vma)) {
                if (arch_needs_pgtable_deposit())
                        zap_deposited_table(tlb->mm, pmd);
                spin_unlock(ptl);
        } else if (is_huge_zero_pmd(orig_pmd)) {
                if (!vma_is_dax(vma) || arch_needs_pgtable_deposit())
                        zap_deposited_table(tlb->mm, pmd);
                spin_unlock(ptl);
        } else {
                struct folio *folio = NULL;
                int flush_needed = 1;

                if (pmd_present(orig_pmd)) {
                        struct page *page = pmd_page(orig_pmd);

                        folio = page_folio(page);
                        folio_remove_rmap_pmd(folio, page, vma);
                        WARN_ON_ONCE(folio_mapcount(folio) < 0);
                        VM_BUG_ON_PAGE(!PageHead(page), page);
                } else if (thp_migration_supported()) {
                        swp_entry_t entry;

                        VM_BUG_ON(!is_pmd_migration_entry(orig_pmd));
                        entry = pmd_to_swp_entry(orig_pmd);
                        folio = pfn_swap_entry_folio(entry);
                        flush_needed = 0;
                } else
                        WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");

                if (folio_test_anon(folio)) {
                        zap_deposited_table(tlb->mm, pmd);
                        add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
                } else {
                        if (arch_needs_pgtable_deposit())
                                zap_deposited_table(tlb->mm, pmd);
                        add_mm_counter(tlb->mm, mm_counter_file(folio),
                                       -HPAGE_PMD_NR);
                }

                spin_unlock(ptl);
                if (flush_needed)
                        tlb_remove_page_size(tlb, &folio->page, HPAGE_PMD_SIZE);
        }
        return 1;
}

#ifndef pmd_move_must_withdraw
static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl,
                                         spinlock_t *old_pmd_ptl,
                                         struct vm_area_struct *vma)
{
        /*
         * With split pmd lock we also need to move preallocated
         * PTE page table if new_pmd is on different PMD page table.
         *
         * We also don't deposit and withdraw tables for file pages.
         */
        return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma);
}
#endif

static pmd_t move_soft_dirty_pmd(pmd_t pmd)
{
#ifdef CONFIG_MEM_SOFT_DIRTY
        if (unlikely(is_pmd_migration_entry(pmd)))
                pmd = pmd_swp_mksoft_dirty(pmd);
        else if (pmd_present(pmd))
                pmd = pmd_mksoft_dirty(pmd);
#endif
        return pmd;
}

static pmd_t clear_uffd_wp_pmd(pmd_t pmd)
{
        if (pmd_present(pmd))
                pmd = pmd_clear_uffd_wp(pmd);
        else if (is_swap_pmd(pmd))
                pmd = pmd_swp_clear_uffd_wp(pmd);

        return pmd;
}

bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
                  unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd)
{
        spinlock_t *old_ptl, *new_ptl;
        pmd_t pmd;
        struct mm_struct *mm = vma->vm_mm;
        bool force_flush = false;

        /*
         * The destination pmd shouldn't be established, free_pgtables()
         * should have released it; but move_page_tables() might have already
         * inserted a page table, if racing against shmem/file collapse.
         */
        if (!pmd_none(*new_pmd)) {
                VM_BUG_ON(pmd_trans_huge(*new_pmd));
                return false;
        }

        /*
         * We don't have to worry about the ordering of src and dst
         * ptlocks because exclusive mmap_lock prevents deadlock.
         */
        old_ptl = __pmd_trans_huge_lock(old_pmd, vma);
        if (old_ptl) {
                new_ptl = pmd_lockptr(mm, new_pmd);
                if (new_ptl != old_ptl)
                        spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
                pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);
                if (pmd_present(pmd))
                        force_flush = true;
                VM_BUG_ON(!pmd_none(*new_pmd));

                if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) {
                        pgtable_t pgtable;
                        pgtable = pgtable_trans_huge_withdraw(mm, old_pmd);
                        pgtable_trans_huge_deposit(mm, new_pmd, pgtable);
                }
                pmd = move_soft_dirty_pmd(pmd);
                if (vma_has_uffd_without_event_remap(vma))
                        pmd = clear_uffd_wp_pmd(pmd);
                set_pmd_at(mm, new_addr, new_pmd, pmd);
                if (force_flush)
                        flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
                if (new_ptl != old_ptl)
                        spin_unlock(new_ptl);
                spin_unlock(old_ptl);
                return true;
        }
        return false;
}

/*
 * Returns
 *  - 0 if PMD could not be locked
 *  - 1 if PMD was locked but protections unchanged and TLB flush unnecessary
 *      or if prot_numa but THP migration is not supported
 *  - HPAGE_PMD_NR if protections changed and TLB flush necessary
 */
int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
                    pmd_t *pmd, unsigned long addr, pgprot_t newprot,
                    unsigned long cp_flags)
{
        struct mm_struct *mm = vma->vm_mm;
        spinlock_t *ptl;
        pmd_t oldpmd, entry;
        bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
        bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
        bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
        int ret = 1;

        tlb_change_page_size(tlb, HPAGE_PMD_SIZE);

        if (prot_numa && !thp_migration_supported())
                return 1;

        ptl = __pmd_trans_huge_lock(pmd, vma);
        if (!ptl)
                return 0;

#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
        if (is_swap_pmd(*pmd)) {
                swp_entry_t entry = pmd_to_swp_entry(*pmd);
                struct folio *folio = pfn_swap_entry_folio(entry);
                pmd_t newpmd;

                VM_BUG_ON(!is_pmd_migration_entry(*pmd));
                if (is_writable_migration_entry(entry)) {
                        /*
                         * A protection check is difficult so
                         * just be safe and disable write
                         */
                        if (folio_test_anon(folio))
                                entry = make_readable_exclusive_migration_entry(swp_offset(entry));
                        else
                                entry = make_readable_migration_entry(swp_offset(entry));
                        newpmd = swp_entry_to_pmd(entry);
                        if (pmd_swp_soft_dirty(*pmd))
                                newpmd = pmd_swp_mksoft_dirty(newpmd);
                } else {
                        newpmd = *pmd;
                }

                if (uffd_wp)
                        newpmd = pmd_swp_mkuffd_wp(newpmd);
                else if (uffd_wp_resolve)
                        newpmd = pmd_swp_clear_uffd_wp(newpmd);
                if (!pmd_same(*pmd, newpmd))
                        set_pmd_at(mm, addr, pmd, newpmd);
                goto unlock;
        }
#endif

        if (prot_numa) {
                struct folio *folio;
                bool toptier;
                /*
                 * Avoid trapping faults against the zero page. The read-only
                 * data is likely to be read-cached on the local CPU and
                 * local/remote hits to the zero page are not interesting.
                 */
                if (is_huge_zero_pmd(*pmd))
                        goto unlock;

                if (pmd_protnone(*pmd))
                        goto unlock;

                folio = pmd_folio(*pmd);
                toptier = node_is_toptier(folio_nid(folio));
                /*
                 * Skip scanning top tier node if normal numa
                 * balancing is disabled
                 */
                if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) &&
                    toptier)
                        goto unlock;

                if (folio_use_access_time(folio))
                        folio_xchg_access_time(folio,
                                               jiffies_to_msecs(jiffies));
        }
        /*
         * In case prot_numa, we are under mmap_read_lock(mm). It's critical
         * to not clear pmd intermittently to avoid race with MADV_DONTNEED
         * which is also under mmap_read_lock(mm):
         *
         *        CPU0:                                CPU1:
         *                                change_huge_pmd(prot_numa=1)
         *                                 pmdp_huge_get_and_clear_notify()
         * madvise_dontneed()
         *  zap_pmd_range()
         *   pmd_trans_huge(*pmd) == 0 (without ptl)
         *   // skip the pmd
         *                                 set_pmd_at();
         *                                 // pmd is re-established
         *
         * The race makes MADV_DONTNEED miss the huge pmd and don't clear it
         * which may break userspace.
         *
         * pmdp_invalidate_ad() is required to make sure we don't miss
         * dirty/young flags set by hardware.
         */
        oldpmd = pmdp_invalidate_ad(vma, addr, pmd);

        entry = pmd_modify(oldpmd, newprot);
        if (uffd_wp)
                entry = pmd_mkuffd_wp(entry);
        else if (uffd_wp_resolve)
                /*
                 * Leave the write bit to be handled by PF interrupt
                 * handler, then things like COW could be properly
                 * handled.
                 */
                entry = pmd_clear_uffd_wp(entry);

        /* See change_pte_range(). */
        if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) &&
            can_change_pmd_writable(vma, addr, entry))
                entry = pmd_mkwrite(entry, vma);

        ret = HPAGE_PMD_NR;
        set_pmd_at(mm, addr, pmd, entry);

        if (huge_pmd_needs_flush(oldpmd, entry))
                tlb_flush_pmd_range(tlb, addr, HPAGE_PMD_SIZE);
unlock:
        spin_unlock(ptl);
        return ret;
}

/*
 * Returns:
 *
 * - 0: if pud leaf changed from under us
 * - 1: if pud can be skipped
 * - HPAGE_PUD_NR: if pud was successfully processed
 */
#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
                    pud_t *pudp, unsigned long addr, pgprot_t newprot,
                    unsigned long cp_flags)
{
        struct mm_struct *mm = vma->vm_mm;
        pud_t oldpud, entry;
        spinlock_t *ptl;

        tlb_change_page_size(tlb, HPAGE_PUD_SIZE);

        /* NUMA balancing doesn't apply to dax */
        if (cp_flags & MM_CP_PROT_NUMA)
                return 1;

        /*
         * Huge entries on userfault-wp only works with anonymous, while we
         * don't have anonymous PUDs yet.
         */
        if (WARN_ON_ONCE(cp_flags & MM_CP_UFFD_WP_ALL))
                return 1;

        ptl = __pud_trans_huge_lock(pudp, vma);
        if (!ptl)
                return 0;

        /*
         * Can't clear PUD or it can race with concurrent zapping.  See
         * change_huge_pmd().
         */
        oldpud = pudp_invalidate(vma, addr, pudp);
        entry = pud_modify(oldpud, newprot);
        set_pud_at(mm, addr, pudp, entry);
        tlb_flush_pud_range(tlb, addr, HPAGE_PUD_SIZE);

        spin_unlock(ptl);
        return HPAGE_PUD_NR;
}
#endif

#ifdef CONFIG_USERFAULTFD
/*
 * The PT lock for src_pmd and dst_vma/src_vma (for reading) are locked by
 * the caller, but it must return after releasing the page_table_lock.
 * Just move the page from src_pmd to dst_pmd if possible.
 * Return zero if succeeded in moving the page, -EAGAIN if it needs to be
 * repeated by the caller, or other errors in case of failure.
 */
int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pmd_t dst_pmdval,
                        struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
                        unsigned long dst_addr, unsigned long src_addr)
{
        pmd_t _dst_pmd, src_pmdval;
        struct page *src_page;
        struct folio *src_folio;
        struct anon_vma *src_anon_vma;
        spinlock_t *src_ptl, *dst_ptl;
        pgtable_t src_pgtable;
        struct mmu_notifier_range range;
        int err = 0;

        src_pmdval = *src_pmd;
        src_ptl = pmd_lockptr(mm, src_pmd);

        lockdep_assert_held(src_ptl);
        vma_assert_locked(src_vma);
        vma_assert_locked(dst_vma);

        /* Sanity checks before the operation */
        if (WARN_ON_ONCE(!pmd_none(dst_pmdval)) || WARN_ON_ONCE(src_addr & ~HPAGE_PMD_MASK) ||
            WARN_ON_ONCE(dst_addr & ~HPAGE_PMD_MASK)) {
                spin_unlock(src_ptl);
                return -EINVAL;
        }

        if (!pmd_trans_huge(src_pmdval)) {
                spin_unlock(src_ptl);
                if (is_pmd_migration_entry(src_pmdval)) {
                        pmd_migration_entry_wait(mm, &src_pmdval);
                        return -EAGAIN;
                }
                return -ENOENT;
        }

        src_page = pmd_page(src_pmdval);

        if (!is_huge_zero_pmd(src_pmdval)) {
                if (unlikely(!PageAnonExclusive(src_page))) {
                        spin_unlock(src_ptl);
                        return -EBUSY;
                }

                src_folio = page_folio(src_page);
                folio_get(src_folio);
        } else
                src_folio = NULL;

        spin_unlock(src_ptl);

        flush_cache_range(src_vma, src_addr, src_addr + HPAGE_PMD_SIZE);
        mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, src_addr,
                                src_addr + HPAGE_PMD_SIZE);
        mmu_notifier_invalidate_range_start(&range);

        if (src_folio) {
                folio_lock(src_folio);

                /*
                 * split_huge_page walks the anon_vma chain without the page
                 * lock. Serialize against it with the anon_vma lock, the page
                 * lock is not enough.
                 */
                src_anon_vma = folio_get_anon_vma(src_folio);
                if (!src_anon_vma) {
                        err = -EAGAIN;
                        goto unlock_folio;
                }
                anon_vma_lock_write(src_anon_vma);
        } else
                src_anon_vma = NULL;

        dst_ptl = pmd_lockptr(mm, dst_pmd);
        double_pt_lock(src_ptl, dst_ptl);
        if (unlikely(!pmd_same(*src_pmd, src_pmdval) ||
                     !pmd_same(*dst_pmd, dst_pmdval))) {
                err = -EAGAIN;
                goto unlock_ptls;
        }
        if (src_folio) {
                if (folio_maybe_dma_pinned(src_folio) ||
                    !PageAnonExclusive(&src_folio->page)) {
                        err = -EBUSY;
                        goto unlock_ptls;
                }

                if (WARN_ON_ONCE(!folio_test_head(src_folio)) ||
                    WARN_ON_ONCE(!folio_test_anon(src_folio))) {
                        err = -EBUSY;
                        goto unlock_ptls;
                }

                src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd);
                /* Folio got pinned from under us. Put it back and fail the move. */
                if (folio_maybe_dma_pinned(src_folio)) {
                        set_pmd_at(mm, src_addr, src_pmd, src_pmdval);
                        err = -EBUSY;
                        goto unlock_ptls;
                }

                folio_move_anon_rmap(src_folio, dst_vma);
                src_folio->index = linear_page_index(dst_vma, dst_addr);

                _dst_pmd = mk_huge_pmd(&src_folio->page, dst_vma->vm_page_prot);
                /* Follow mremap() behavior and treat the entry dirty after the move */
                _dst_pmd = pmd_mkwrite(pmd_mkdirty(_dst_pmd), dst_vma);
        } else {
                src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd);
                _dst_pmd = mk_huge_pmd(src_page, dst_vma->vm_page_prot);
        }
        set_pmd_at(mm, dst_addr, dst_pmd, _dst_pmd);

        src_pgtable = pgtable_trans_huge_withdraw(mm, src_pmd);
        pgtable_trans_huge_deposit(mm, dst_pmd, src_pgtable);
unlock_ptls:
        double_pt_unlock(src_ptl, dst_ptl);
        if (src_anon_vma) {
                anon_vma_unlock_write(src_anon_vma);
                put_anon_vma(src_anon_vma);
        }
unlock_folio:
        /* unblock rmap walks */
        if (src_folio)
                folio_unlock(src_folio);
        mmu_notifier_invalidate_range_end(&range);
        if (src_folio)
                folio_put(src_folio);
        return err;
}
#endif /* CONFIG_USERFAULTFD */

/*
 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise.
 *
 * Note that if it returns page table lock pointer, this routine returns without
 * unlocking page table lock. So callers must unlock it.
 */
spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma)
{
        spinlock_t *ptl;
        ptl = pmd_lock(vma->vm_mm, pmd);
        if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) ||
                        pmd_devmap(*pmd)))
                return ptl;
        spin_unlock(ptl);
        return NULL;
}

/*
 * Returns page table lock pointer if a given pud maps a thp, NULL otherwise.
 *
 * Note that if it returns page table lock pointer, this routine returns without
 * unlocking page table lock. So callers must unlock it.
 */
spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma)
{
        spinlock_t *ptl;

        ptl = pud_lock(vma->vm_mm, pud);
        if (likely(pud_trans_huge(*pud) || pud_devmap(*pud)))
                return ptl;
        spin_unlock(ptl);
        return NULL;
}

#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
                 pud_t *pud, unsigned long addr)
{
        spinlock_t *ptl;
        pud_t orig_pud;

        ptl = __pud_trans_huge_lock(pud, vma);
        if (!ptl)
                return 0;

        orig_pud = pudp_huge_get_and_clear_full(vma, addr, pud, tlb->fullmm);
        arch_check_zapped_pud(vma, orig_pud);
        tlb_remove_pud_tlb_entry(tlb, pud, addr);
        if (!vma_is_dax(vma) && vma_is_special_huge(vma)) {
                spin_unlock(ptl);
                /* No zero page support yet */
        } else {
                struct page *page = NULL;
                struct folio *folio;

                /* No support for anonymous PUD pages or migration yet */
                VM_WARN_ON_ONCE(vma_is_anonymous(vma) ||
                                !pud_present(orig_pud));

                page = pud_page(orig_pud);
                folio = page_folio(page);
                folio_remove_rmap_pud(folio, page, vma);
                add_mm_counter(tlb->mm, mm_counter_file(folio), -HPAGE_PUD_NR);

                spin_unlock(ptl);
                tlb_remove_page_size(tlb, page, HPAGE_PUD_SIZE);
        }
        return 1;
}

static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud,
                unsigned long haddr)
{
        struct folio *folio;
        struct page *page;
        pud_t old_pud;

        VM_BUG_ON(haddr & ~HPAGE_PUD_MASK);
        VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
        VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma);
        VM_BUG_ON(!pud_trans_huge(*pud) && !pud_devmap(*pud));

        count_vm_event(THP_SPLIT_PUD);

        old_pud = pudp_huge_clear_flush(vma, haddr, pud);

        if (!vma_is_dax(vma))
                return;

        page = pud_page(old_pud);
        folio = page_folio(page);

        if (!folio_test_dirty(folio) && pud_dirty(old_pud))
                folio_mark_dirty(folio);
        if (!folio_test_referenced(folio) && pud_young(old_pud))
                folio_set_referenced(folio);
        folio_remove_rmap_pud(folio, page, vma);
        folio_put(folio);
        add_mm_counter(vma->vm_mm, mm_counter_file(folio),
                -HPAGE_PUD_NR);
}

void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
                unsigned long address)
{
        spinlock_t *ptl;
        struct mmu_notifier_range range;

        mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
                                address & HPAGE_PUD_MASK,
                                (address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE);
        mmu_notifier_invalidate_range_start(&range);
        ptl = pud_lock(vma->vm_mm, pud);
        if (unlikely(!pud_trans_huge(*pud) && !pud_devmap(*pud)))
                goto out;
        __split_huge_pud_locked(vma, pud, range.start);

out:
        spin_unlock(ptl);
        mmu_notifier_invalidate_range_end(&range);
}
#else
void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
                unsigned long address)
{
}
#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */

static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
                unsigned long haddr, pmd_t *pmd)
{
        struct mm_struct *mm = vma->vm_mm;
        pgtable_t pgtable;
        pmd_t _pmd, old_pmd;
        unsigned long addr;
        pte_t *pte;
        int i;

        /*
         * Leave pmd empty until pte is filled note that it is fine to delay
         * notification until mmu_notifier_invalidate_range_end() as we are
         * replacing a zero pmd write protected page with a zero pte write
         * protected page.
         *
         * See Documentation/mm/mmu_notifier.rst
         */
        old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd);

        pgtable = pgtable_trans_huge_withdraw(mm, pmd);
        pmd_populate(mm, &_pmd, pgtable);

        pte = pte_offset_map(&_pmd, haddr);
        VM_BUG_ON(!pte);
        for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) {
                pte_t entry;

                entry = pfn_pte(my_zero_pfn(addr), vma->vm_page_prot);
                entry = pte_mkspecial(entry);
                if (pmd_uffd_wp(old_pmd))
                        entry = pte_mkuffd_wp(entry);
                VM_BUG_ON(!pte_none(ptep_get(pte)));
                set_pte_at(mm, addr, pte, entry);
                pte++;
        }
        pte_unmap(pte - 1);
        smp_wmb(); /* make pte visible before pmd */
        pmd_populate(mm, pmd, pgtable);
}

static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
                unsigned long haddr, bool freeze)
{
        struct mm_struct *mm = vma->vm_mm;
        struct folio *folio;
        struct page *page;
        pgtable_t pgtable;
        pmd_t old_pmd, _pmd;
        bool young, write, soft_dirty, pmd_migration = false, uffd_wp = false;
        bool anon_exclusive = false, dirty = false;
        unsigned long addr;
        pte_t *pte;
        int i;

        VM_BUG_ON(haddr & ~HPAGE_PMD_MASK);
        VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
        VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma);
        VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd)
                                && !pmd_devmap(*pmd));

        count_vm_event(THP_SPLIT_PMD);

        if (!vma_is_anonymous(vma)) {
                old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd);
                /*
                 * We are going to unmap this huge page. So
                 * just go ahead and zap it
                 */
                if (arch_needs_pgtable_deposit())
                        zap_deposited_table(mm, pmd);
                if (!vma_is_dax(vma) && vma_is_special_huge(vma))
                        return;
                if (unlikely(is_pmd_migration_entry(old_pmd))) {
                        swp_entry_t entry;

                        entry = pmd_to_swp_entry(old_pmd);
                        folio = pfn_swap_entry_folio(entry);
                } else if (is_huge_zero_pmd(old_pmd)) {
                        return;
                } else {
                        page = pmd_page(old_pmd);
                        folio = page_folio(page);
                        if (!folio_test_dirty(folio) && pmd_dirty(old_pmd))
                                folio_mark_dirty(folio);
                        if (!folio_test_referenced(folio) && pmd_young(old_pmd))
                                folio_set_referenced(folio);
                        folio_remove_rmap_pmd(folio, page, vma);
                        folio_put(folio);
                }
                add_mm_counter(mm, mm_counter_file(folio), -HPAGE_PMD_NR);
                return;
        }

        if (is_huge_zero_pmd(*pmd)) {
                /*
                 * FIXME: Do we want to invalidate secondary mmu by calling
                 * mmu_notifier_arch_invalidate_secondary_tlbs() see comments below
                 * inside __split_huge_pmd() ?
                 *
                 * We are going from a zero huge page write protected to zero
                 * small page also write protected so it does not seems useful
                 * to invalidate secondary mmu at this time.
                 */
                return __split_huge_zero_page_pmd(vma, haddr, pmd);
        }

        pmd_migration = is_pmd_migration_entry(*pmd);
        if (unlikely(pmd_migration)) {
                swp_entry_t entry;

                old_pmd = *pmd;
                entry = pmd_to_swp_entry(old_pmd);
                page = pfn_swap_entry_to_page(entry);
                write = is_writable_migration_entry(entry);
                if (PageAnon(page))
                        anon_exclusive = is_readable_exclusive_migration_entry(entry);
                young = is_migration_entry_young(entry);
                dirty = is_migration_entry_dirty(entry);
                soft_dirty = pmd_swp_soft_dirty(old_pmd);
                uffd_wp = pmd_swp_uffd_wp(old_pmd);
        } else {
                /*
                 * Up to this point the pmd is present and huge and userland has
                 * the whole access to the hugepage during the split (which
                 * happens in place). If we overwrite the pmd with the not-huge
                 * version pointing to the pte here (which of course we could if
                 * all CPUs were bug free), userland could trigger a small page
                 * size TLB miss on the small sized TLB while the hugepage TLB
                 * entry is still established in the huge TLB. Some CPU doesn't
                 * like that. See
                 * http://support.amd.com/TechDocs/41322_10h_Rev_Gd.pdf, Erratum
                 * 383 on page 105. Intel should be safe but is also warns that
                 * it's only safe if the permission and cache attributes of the
                 * two entries loaded in the two TLB is identical (which should
                 * be the case here). But it is generally safer to never allow
                 * small and huge TLB entries for the same virtual address to be
                 * loaded simultaneously. So instead of doing "pmd_populate();
                 * flush_pmd_tlb_range();" we first mark the current pmd
                 * notpresent (atomically because here the pmd_trans_huge must
                 * remain set at all times on the pmd until the split is
                 * complete for this pmd), then we flush the SMP TLB and finally
                 * we write the non-huge version of the pmd entry with
                 * pmd_populate.
                 */
                old_pmd = pmdp_invalidate(vma, haddr, pmd);
                page = pmd_page(old_pmd);
                folio = page_folio(page);
                if (pmd_dirty(old_pmd)) {
                        dirty = true;
                        folio_set_dirty(folio);
                }
                write = pmd_write(old_pmd);
                young = pmd_young(old_pmd);
                soft_dirty = pmd_soft_dirty(old_pmd);
                uffd_wp = pmd_uffd_wp(old_pmd);

                VM_WARN_ON_FOLIO(!folio_ref_count(folio), folio);
                VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);

                /*
                 * Without "freeze", we'll simply split the PMD, propagating the
                 * PageAnonExclusive() flag for each PTE by setting it for
                 * each subpage -- no need to (temporarily) clear.
                 *
                 * With "freeze" we want to replace mapped pages by
                 * migration entries right away. This is only possible if we
                 * managed to clear PageAnonExclusive() -- see
                 * set_pmd_migration_entry().
                 *
                 * In case we cannot clear PageAnonExclusive(), split the PMD
                 * only and let try_to_migrate_one() fail later.
                 *
                 * See folio_try_share_anon_rmap_pmd(): invalidate PMD first.
                 */
                anon_exclusive = PageAnonExclusive(page);
                if (freeze && anon_exclusive &&
                    folio_try_share_anon_rmap_pmd(folio, page))
                        freeze = false;
                if (!freeze) {
                        rmap_t rmap_flags = RMAP_NONE;

                        folio_ref_add(folio, HPAGE_PMD_NR - 1);
                        if (anon_exclusive)
                                rmap_flags |= RMAP_EXCLUSIVE;
                        folio_add_anon_rmap_ptes(folio, page, HPAGE_PMD_NR,
                                                 vma, haddr, rmap_flags);
                }
        }

        /*
         * Withdraw the table only after we mark the pmd entry invalid.
         * This's critical for some architectures (Power).
         */
        pgtable = pgtable_trans_huge_withdraw(mm, pmd);
        pmd_populate(mm, &_pmd, pgtable);

        pte = pte_offset_map(&_pmd, haddr);
        VM_BUG_ON(!pte);

        /*
         * Note that NUMA hinting access restrictions are not transferred to
         * avoid any possibility of altering permissions across VMAs.
         */
        if (freeze || pmd_migration) {
                for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) {
                        pte_t entry;
                        swp_entry_t swp_entry;

                        if (write)
                                swp_entry = make_writable_migration_entry(
                                                        page_to_pfn(page + i));
                        else if (anon_exclusive)
                                swp_entry = make_readable_exclusive_migration_entry(
                                                        page_to_pfn(page + i));
                        else
                                swp_entry = make_readable_migration_entry(
                                                        page_to_pfn(page + i));
                        if (young)
                                swp_entry = make_migration_entry_young(swp_entry);
                        if (dirty)
                                swp_entry = make_migration_entry_dirty(swp_entry);
                        entry = swp_entry_to_pte(swp_entry);
                        if (soft_dirty)
                                entry = pte_swp_mksoft_dirty(entry);
                        if (uffd_wp)
                                entry = pte_swp_mkuffd_wp(entry);

                        VM_WARN_ON(!pte_none(ptep_get(pte + i)));
                        set_pte_at(mm, addr, pte + i, entry);
                }
        } else {
                pte_t entry;

                entry = mk_pte(page, READ_ONCE(vma->vm_page_prot));
                if (write)
                        entry = pte_mkwrite(entry, vma);
                if (!young)
                        entry = pte_mkold(entry);
                /* NOTE: this may set soft-dirty too on some archs */
                if (dirty)
                        entry = pte_mkdirty(entry);
                if (soft_dirty)
                        entry = pte_mksoft_dirty(entry);
                if (uffd_wp)
                        entry = pte_mkuffd_wp(entry);

                for (i = 0; i < HPAGE_PMD_NR; i++)
                        VM_WARN_ON(!pte_none(ptep_get(pte + i)));

                set_ptes(mm, haddr, pte, entry, HPAGE_PMD_NR);
        }
        pte_unmap(pte);

        if (!pmd_migration)
                folio_remove_rmap_pmd(folio, page, vma);
        if (freeze)
                put_page(page);

        smp_wmb(); /* make pte visible before pmd */
        pmd_populate(mm, pmd, pgtable);
}

void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address,
                           pmd_t *pmd, bool freeze, struct folio *folio)
{
        VM_WARN_ON_ONCE(folio && !folio_test_pmd_mappable(folio));
        VM_WARN_ON_ONCE(!IS_ALIGNED(address, HPAGE_PMD_SIZE));
        VM_WARN_ON_ONCE(folio && !folio_test_locked(folio));
        VM_BUG_ON(freeze && !folio);

        /*
         * When the caller requests to set up a migration entry, we
         * require a folio to check the PMD against. Otherwise, there
         * is a risk of replacing the wrong folio.
         */
        if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd) ||
            is_pmd_migration_entry(*pmd)) {
                if (folio && folio != pmd_folio(*pmd))
                        return;
                __split_huge_pmd_locked(vma, pmd, address, freeze);
        }
}

void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
                unsigned long address, bool freeze, struct folio *folio)
{
        spinlock_t *ptl;
        struct mmu_notifier_range range;

        mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
                                address & HPAGE_PMD_MASK,
                                (address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE);
        mmu_notifier_invalidate_range_start(&range);
        ptl = pmd_lock(vma->vm_mm, pmd);
        split_huge_pmd_locked(vma, range.start, pmd, freeze, folio);
        spin_unlock(ptl);
        mmu_notifier_invalidate_range_end(&range);
}

void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
                bool freeze, struct folio *folio)
{
        pmd_t *pmd = mm_find_pmd(vma->vm_mm, address);

        if (!pmd)
                return;

        __split_huge_pmd(vma, pmd, address, freeze, folio);
}

static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address)
{
        /*
         * If the new address isn't hpage aligned and it could previously
         * contain an hugepage: check if we need to split an huge pmd.
         */
        if (!IS_ALIGNED(address, HPAGE_PMD_SIZE) &&
            range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PMD_SIZE),
                         ALIGN(address, HPAGE_PMD_SIZE)))
                split_huge_pmd_address(vma, address, false, NULL);
}

void vma_adjust_trans_huge(struct vm_area_struct *vma,
                           unsigned long start,
                           unsigned long end,
                           struct vm_area_struct *next)
{
        /* Check if we need to split start first. */
        split_huge_pmd_if_needed(vma, start);

        /* Check if we need to split end next. */
        split_huge_pmd_if_needed(vma, end);

        /* If we're incrementing next->vm_start, we might need to split it. */
        if (next)
                split_huge_pmd_if_needed(next, end);
}

static void unmap_folio(struct folio *folio)
{
        enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SYNC |
                TTU_BATCH_FLUSH;

        VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);

        if (folio_test_pmd_mappable(folio))
                ttu_flags |= TTU_SPLIT_HUGE_PMD;

        /*
         * Anon pages need migration entries to preserve them, but file
         * pages can simply be left unmapped, then faulted back on demand.
         * If that is ever changed (perhaps for mlock), update remap_page().
         */
        if (folio_test_anon(folio))
                try_to_migrate(folio, ttu_flags);
        else
                try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK);

        try_to_unmap_flush();
}

static bool __discard_anon_folio_pmd_locked(struct vm_area_struct *vma,
                                            unsigned long addr, pmd_t *pmdp,
                                            struct folio *folio)
{
        struct mm_struct *mm = vma->vm_mm;
        int ref_count, map_count;
        pmd_t orig_pmd = *pmdp;

        if (pmd_dirty(orig_pmd))
                folio_set_dirty(folio);
        if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) {
                folio_set_swapbacked(folio);
                return false;
        }

        orig_pmd = pmdp_huge_clear_flush(vma, addr, pmdp);

        /*
         * Syncing against concurrent GUP-fast:
         * - clear PMD; barrier; read refcount
         * - inc refcount; barrier; read PMD
         */
        smp_mb();

        ref_count = folio_ref_count(folio);
        map_count = folio_mapcount(folio);

        /*
         * Order reads for folio refcount and dirty flag
         * (see comments in __remove_mapping()).
         */
        smp_rmb();

        /*
         * If the folio or its PMD is redirtied at this point, or if there
         * are unexpected references, we will give up to discard this folio
         * and remap it.
         *
         * The only folio refs must be one from isolation plus the rmap(s).
         */
        if (pmd_dirty(orig_pmd))
                folio_set_dirty(folio);
        if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) {
                folio_set_swapbacked(folio);
                set_pmd_at(mm, addr, pmdp, orig_pmd);
                return false;
        }

        if (ref_count != map_count + 1) {
                set_pmd_at(mm, addr, pmdp, orig_pmd);
                return false;
        }

        folio_remove_rmap_pmd(folio, pmd_page(orig_pmd), vma);
        zap_deposited_table(mm, pmdp);
        add_mm_counter(mm, MM_ANONPAGES, -HPAGE_PMD_NR);
        if (vma->vm_flags & VM_LOCKED)
                mlock_drain_local();
        folio_put(folio);

        return true;
}

bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr,
                           pmd_t *pmdp, struct folio *folio)
{
        VM_WARN_ON_FOLIO(!folio_test_pmd_mappable(folio), folio);
        VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
        VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
        VM_WARN_ON_FOLIO(folio_test_swapbacked(folio), folio);
        VM_WARN_ON_ONCE(!IS_ALIGNED(addr, HPAGE_PMD_SIZE));

        return __discard_anon_folio_pmd_locked(vma, addr, pmdp, folio);
}

static void remap_page(struct folio *folio, unsigned long nr, int flags)
{
        int i = 0;

        /* If unmap_folio() uses try_to_migrate() on file, remove this check */
        if (!folio_test_anon(folio))
                return;
        for (;;) {
                remove_migration_ptes(folio, folio, RMP_LOCKED | flags);
                i += folio_nr_pages(folio);
                if (i >= nr)
                        break;
                folio = folio_next(folio);
        }
}

static void lru_add_split_folio(struct folio *folio, struct folio *new_folio,
                struct lruvec *lruvec, struct list_head *list)
{
        VM_BUG_ON_FOLIO(folio_test_lru(new_folio), folio);
        lockdep_assert_held(&lruvec->lru_lock);

        if (list) {
                /* page reclaim is reclaiming a huge page */
                VM_WARN_ON(folio_test_lru(folio));
                folio_get(new_folio);
                list_add_tail(&new_folio->lru, list);
        } else {
                /* head is still on lru (and we have it frozen) */
                VM_WARN_ON(!folio_test_lru(folio));
                if (folio_test_unevictable(folio))
                        new_folio->mlock_count = 0;
                else
                        list_add_tail(&new_folio->lru, &folio->lru);
                folio_set_lru(new_folio);
        }
}

/* Racy check whether the huge page can be split */
bool can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins)
{
        int extra_pins;

        /* Additional pins from page cache */
        if (folio_test_anon(folio))
                extra_pins = folio_test_swapcache(folio) ?
                                folio_nr_pages(folio) : 0;
        else
                extra_pins = folio_nr_pages(folio);
        if (pextra_pins)
                *pextra_pins = extra_pins;
        return folio_mapcount(folio) == folio_ref_count(folio) - extra_pins -
                                        caller_pins;
}

/*
 * It splits @folio into @new_order folios and copies the @folio metadata to
 * all the resulting folios.
 */
static void __split_folio_to_order(struct folio *folio, int old_order,
                int new_order)
{
        long new_nr_pages = 1 << new_order;
        long nr_pages = 1 << old_order;
        long i;

        /*
         * Skip the first new_nr_pages, since the new folio from them have all
         * the flags from the original folio.
         */
        for (i = new_nr_pages; i < nr_pages; i += new_nr_pages) {
                struct page *new_head = &folio->page + i;

                /*
                 * Careful: new_folio is not a "real" folio before we cleared PageTail.
                 * Don't pass it around before clear_compound_head().
                 */
                struct folio *new_folio = (struct folio *)new_head;

                VM_BUG_ON_PAGE(atomic_read(&new_folio->_mapcount) != -1, new_head);

                /*
                 * Clone page flags before unfreezing refcount.
                 *
                 * After successful get_page_unless_zero() might follow flags change,
                 * for example lock_page() which set PG_waiters.
                 *
                 * Note that for mapped sub-pages of an anonymous THP,
                 * PG_anon_exclusive has been cleared in unmap_folio() and is stored in
                 * the migration entry instead from where remap_page() will restore it.
                 * We can still have PG_anon_exclusive set on effectively unmapped and
                 * unreferenced sub-pages of an anonymous THP: we can simply drop
                 * PG_anon_exclusive (-> PG_mappedtodisk) for these here.
                 */
                new_folio->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
                new_folio->flags |= (folio->flags &
                                ((1L << PG_referenced) |
                                 (1L << PG_swapbacked) |
                                 (1L << PG_swapcache) |
                                 (1L << PG_mlocked) |
                                 (1L << PG_uptodate) |
                                 (1L << PG_active) |
                                 (1L << PG_workingset) |
                                 (1L << PG_locked) |
                                 (1L << PG_unevictable) |
#ifdef CONFIG_ARCH_USES_PG_ARCH_2
                                 (1L << PG_arch_2) |
#endif
#ifdef CONFIG_ARCH_USES_PG_ARCH_3
                                 (1L << PG_arch_3) |
#endif
                                 (1L << PG_dirty) |
                                 LRU_GEN_MASK | LRU_REFS_MASK));

                new_folio->mapping = folio->mapping;
                new_folio->index = folio->index + i;

                /*
                 * page->private should not be set in tail pages. Fix up and warn once
                 * if private is unexpectedly set.
                 */
                if (unlikely(new_folio->private)) {
                        VM_WARN_ON_ONCE_PAGE(true, new_head);
                        new_folio->private = NULL;
                }

                if (folio_test_swapcache(folio))
                        new_folio->swap.val = folio->swap.val + i;

                /* Page flags must be visible before we make the page non-compound. */
                smp_wmb();

                /*
                 * Clear PageTail before unfreezing page refcount.
                 *
                 * After successful get_page_unless_zero() might follow put_page()
                 * which needs correct compound_head().
                 */
                clear_compound_head(new_head);
                if (new_order) {
                        prep_compound_page(new_head, new_order);
                        folio_set_large_rmappable(new_folio);
                }

                if (folio_test_young(folio))
                        folio_set_young(new_folio);
                if (folio_test_idle(folio))
                        folio_set_idle(new_folio);
#ifdef CONFIG_MEMCG
                new_folio->memcg_data = folio->memcg_data;
#endif

                folio_xchg_last_cpupid(new_folio, folio_last_cpupid(folio));
        }

        if (new_order)
                folio_set_order(folio, new_order);
        else
                ClearPageCompound(&folio->page);
}

/*
 * It splits an unmapped @folio to lower order smaller folios in two ways.
 * @folio: the to-be-split folio
 * @new_order: the smallest order of the after split folios (since buddy
 *             allocator like split generates folios with orders from @folio's
 *             order - 1 to new_order).
 * @split_at: in buddy allocator like split, the folio containing @split_at
 *            will be split until its order becomes @new_order.
 * @lock_at: the folio containing @lock_at is left locked for caller.
 * @list: the after split folios will be added to @list if it is not NULL,
 *        otherwise to LRU lists.
 * @end: the end of the file @folio maps to. -1 if @folio is anonymous memory.
 * @xas: xa_state pointing to folio->mapping->i_pages and locked by caller
 * @mapping: @folio->mapping
 * @uniform_split: if the split is uniform or not (buddy allocator like split)
 *
 *
 * 1. uniform split: the given @folio into multiple @new_order small folios,
 *    where all small folios have the same order. This is done when
 *    uniform_split is true.
 * 2. buddy allocator like (non-uniform) split: the given @folio is split into
 *    half and one of the half (containing the given page) is split into half
 *    until the given @page's order becomes @new_order. This is done when
 *    uniform_split is false.
 *
 * The high level flow for these two methods are:
 * 1. uniform split: a single __split_folio_to_order() is called to split the
 *    @folio into @new_order, then we traverse all the resulting folios one by
 *    one in PFN ascending order and perform stats, unfreeze, adding to list,
 *    and file mapping index operations.
 * 2. non-uniform split: in general, folio_order - @new_order calls to
 *    __split_folio_to_order() are made in a for loop to split the @folio
 *    to one lower order at a time. The resulting small folios are processed
 *    like what is done during the traversal in 1, except the one containing
 *    @page, which is split in next for loop.
 *
 * After splitting, the caller's folio reference will be transferred to the
 * folio containing @page. The other folios may be freed if they are not mapped.
 *
 * In terms of locking, after splitting,
 * 1. uniform split leaves @page (or the folio contains it) locked;
 * 2. buddy allocator like (non-uniform) split leaves @folio locked.
 *
 *
 * For !uniform_split, when -ENOMEM is returned, the original folio might be
 * split. The caller needs to check the input folio.
 */
static int __split_unmapped_folio(struct folio *folio, int new_order,
                struct page *split_at, struct page *lock_at,
                struct list_head *list, pgoff_t end,
                struct xa_state *xas, struct address_space *mapping,
                bool uniform_split)
{
        struct lruvec *lruvec;
        struct address_space *swap_cache = NULL;
        struct folio *origin_folio = folio;
        struct folio *next_folio = folio_next(folio);
        struct folio *new_folio;
        struct folio *next;
        int order = folio_order(folio);
        int split_order;
        int start_order = uniform_split ? new_order : order - 1;
        int nr_dropped = 0;
        int ret = 0;
        bool stop_split = false;

        if (folio_test_swapcache(folio)) {
                VM_BUG_ON(mapping);

                /* a swapcache folio can only be uniformly split to order-0 */
                if (!uniform_split || new_order != 0)
                        return -EINVAL;

                swap_cache = swap_address_space(folio->swap);
                xa_lock(&swap_cache->i_pages);
        }

        if (folio_test_anon(folio))
                mod_mthp_stat(order, MTHP_STAT_NR_ANON, -1);

        /* lock lru list/PageCompound, ref frozen by page_ref_freeze */
        lruvec = folio_lruvec_lock(folio);

        folio_clear_has_hwpoisoned(folio);

        /*
         * split to new_order one order at a time. For uniform split,
         * folio is split to new_order directly.
         */
        for (split_order = start_order;
             split_order >= new_order && !stop_split;
             split_order--) {
                int old_order = folio_order(folio);
                struct folio *release;
                struct folio *end_folio = folio_next(folio);

                /* order-1 anonymous folio is not supported */
                if (folio_test_anon(folio) && split_order == 1)
                        continue;
                if (uniform_split && split_order != new_order)
                        continue;

                if (mapping) {
                        /*
                         * uniform split has xas_split_alloc() called before
                         * irq is disabled to allocate enough memory, whereas
                         * non-uniform split can handle ENOMEM.
                         */
                        if (uniform_split)
                                xas_split(xas, folio, old_order);
                        else {
                                xas_set_order(xas, folio->index, split_order);
                                xas_try_split(xas, folio, old_order);
                                if (xas_error(xas)) {
                                        ret = xas_error(xas);
                                        stop_split = true;
                                        goto after_split;
                                }
                        }
                }

                folio_split_memcg_refs(folio, old_order, split_order);
                split_page_owner(&folio->page, old_order, split_order);
                pgalloc_tag_split(folio, old_order, split_order);

                __split_folio_to_order(folio, old_order, split_order);

after_split:
                /*
                 * Iterate through after-split folios and perform related
                 * operations. But in buddy allocator like split, the folio
                 * containing the specified page is skipped until its order
                 * is new_order, since the folio will be worked on in next
                 * iteration.
                 */
                for (release = folio; release != end_folio; release = next) {
                        next = folio_next(release);
                        /*
                         * for buddy allocator like split, the folio containing
                         * page will be split next and should not be released,
                         * until the folio's order is new_order or stop_split
                         * is set to true by the above xas_split() failure.
                         */
                        if (release == page_folio(split_at)) {
                                folio = release;
                                if (split_order != new_order && !stop_split)
                                        continue;
                        }
                        if (folio_test_anon(release)) {
                                mod_mthp_stat(folio_order(release),
                                                MTHP_STAT_NR_ANON, 1);
                        }

                        /*
                         * origin_folio should be kept frozon until page cache
                         * entries are updated with all the other after-split
                         * folios to prevent others seeing stale page cache
                         * entries.
                         */
                        if (release == origin_folio)
                                continue;

                        folio_ref_unfreeze(release, 1 +
                                        ((mapping || swap_cache) ?
                                                folio_nr_pages(release) : 0));

                        lru_add_split_folio(origin_folio, release, lruvec,
                                        list);

                        /* Some pages can be beyond EOF: drop them from cache */
                        if (release->index >= end) {
                                if (shmem_mapping(mapping))
                                        nr_dropped += folio_nr_pages(release);
                                else if (folio_test_clear_dirty(release))
                                        folio_account_cleaned(release,
                                                inode_to_wb(mapping->host));
                                __filemap_remove_folio(release, NULL);
                                folio_put_refs(release, folio_nr_pages(release));
                        } else if (mapping) {
                                __xa_store(&mapping->i_pages,
                                                release->index, release, 0);
                        } else if (swap_cache) {
                                __xa_store(&swap_cache->i_pages,
                                                swap_cache_index(release->swap),
                                                release, 0);
                        }
                }
        }

        /*
         * Unfreeze origin_folio only after all page cache entries, which used
         * to point to it, have been updated with new folios. Otherwise,
         * a parallel folio_try_get() can grab origin_folio and its caller can
         * see stale page cache entries.
         */
        folio_ref_unfreeze(origin_folio, 1 +
                ((mapping || swap_cache) ? folio_nr_pages(origin_folio) : 0));

        unlock_page_lruvec(lruvec);

        if (swap_cache)
                xa_unlock(&swap_cache->i_pages);
        if (mapping)
                xa_unlock(&mapping->i_pages);

        /* Caller disabled irqs, so they are still disabled here */
        local_irq_enable();

        if (nr_dropped)
                shmem_uncharge(mapping->host, nr_dropped);

        remap_page(origin_folio, 1 << order,
                        folio_test_anon(origin_folio) ?
                                RMP_USE_SHARED_ZEROPAGE : 0);

        /*
         * At this point, folio should contain the specified page.
         * For uniform split, it is left for caller to unlock.
         * For buddy allocator like split, the first after-split folio is left
         * for caller to unlock.
         */
        for (new_folio = origin_folio; new_folio != next_folio; new_folio = next) {
                next = folio_next(new_folio);
                if (new_folio == page_folio(lock_at))
                        continue;

                folio_unlock(new_folio);
                /*
                 * Subpages may be freed if there wasn't any mapping
                 * like if add_to_swap() is running on a lru page that
                 * had its mapping zapped. And freeing these pages
                 * requires taking the lru_lock so we do the put_page
                 * of the tail pages after the split is complete.
                 */
                free_page_and_swap_cache(&new_folio->page);
        }
        return ret;
}

bool non_uniform_split_supported(struct folio *folio, unsigned int new_order,
                bool warns)
{
        if (folio_test_anon(folio)) {
                /* order-1 is not supported for anonymous THP. */
                VM_WARN_ONCE(warns && new_order == 1,
                                "Cannot split to order-1 folio");
                return new_order != 1;
        } else if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) &&
            !mapping_large_folio_support(folio->mapping)) {
                /*
                 * No split if the file system does not support large folio.
                 * Note that we might still have THPs in such mappings due to
                 * CONFIG_READ_ONLY_THP_FOR_FS. But in that case, the mapping
                 * does not actually support large folios properly.
                 */
                VM_WARN_ONCE(warns,
                        "Cannot split file folio to non-0 order");
                return false;
        }

        /* Only swapping a whole PMD-mapped folio is supported */
        if (folio_test_swapcache(folio)) {
                VM_WARN_ONCE(warns,
                        "Cannot split swapcache folio to non-0 order");
                return false;
        }

        return true;
}

/* See comments in non_uniform_split_supported() */
bool uniform_split_supported(struct folio *folio, unsigned int new_order,
                bool warns)
{
        if (folio_test_anon(folio)) {
                VM_WARN_ONCE(warns && new_order == 1,
                                "Cannot split to order-1 folio");
                return new_order != 1;
        } else  if (new_order) {
                if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) &&
                    !mapping_large_folio_support(folio->mapping)) {
                        VM_WARN_ONCE(warns,
                                "Cannot split file folio to non-0 order");
                        return false;
                }
        }

        if (new_order && folio_test_swapcache(folio)) {
                VM_WARN_ONCE(warns,
                        "Cannot split swapcache folio to non-0 order");
                return false;
        }

        return true;
}

/*
 * __folio_split: split a folio at @split_at to a @new_order folio
 * @folio: folio to split
 * @new_order: the order of the new folio
 * @split_at: a page within the new folio
 * @lock_at: a page within @folio to be left locked to caller
 * @list: after-split folios will be put on it if non NULL
 * @uniform_split: perform uniform split or not (non-uniform split)
 *
 * It calls __split_unmapped_folio() to perform uniform and non-uniform split.
 * It is in charge of checking whether the split is supported or not and
 * preparing @folio for __split_unmapped_folio().
 *
 * return: 0: successful, <0 failed (if -ENOMEM is returned, @folio might be
 * split but not to @new_order, the caller needs to check)
 */
static int __folio_split(struct folio *folio, unsigned int new_order,
                struct page *split_at, struct page *lock_at,
                struct list_head *list, bool uniform_split)
{
        struct deferred_split *ds_queue = get_deferred_split_queue(folio);
        XA_STATE(xas, &folio->mapping->i_pages, folio->index);
        bool is_anon = folio_test_anon(folio);
        struct address_space *mapping = NULL;
        struct anon_vma *anon_vma = NULL;
        int order = folio_order(folio);
        int extra_pins, ret;
        pgoff_t end;
        bool is_hzp;

        VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
        VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);

        if (folio != page_folio(split_at) || folio != page_folio(lock_at))
                return -EINVAL;

        if (new_order >= folio_order(folio))
                return -EINVAL;

        if (uniform_split && !uniform_split_supported(folio, new_order, true))
                return -EINVAL;

        if (!uniform_split &&
            !non_uniform_split_supported(folio, new_order, true))
                return -EINVAL;

        is_hzp = is_huge_zero_folio(folio);
        if (is_hzp) {
                pr_warn_ratelimited("Called split_huge_page for huge zero page\n");
                return -EBUSY;
        }

        if (folio_test_writeback(folio))
                return -EBUSY;

        if (is_anon) {
                /*
                 * The caller does not necessarily hold an mmap_lock that would
                 * prevent the anon_vma disappearing so we first we take a
                 * reference to it and then lock the anon_vma for write. This
                 * is similar to folio_lock_anon_vma_read except the write lock
                 * is taken to serialise against parallel split or collapse
                 * operations.
                 */
                anon_vma = folio_get_anon_vma(folio);
                if (!anon_vma) {
                        ret = -EBUSY;
                        goto out;
                }
                end = -1;
                mapping = NULL;
                anon_vma_lock_write(anon_vma);
        } else {
                unsigned int min_order;
                gfp_t gfp;

                mapping = folio->mapping;

                /* Truncated ? */
                /*
                 * TODO: add support for large shmem folio in swap cache.
                 * When shmem is in swap cache, mapping is NULL and
                 * folio_test_swapcache() is true.
                 */
                if (!mapping) {
                        ret = -EBUSY;
                        goto out;
                }

                min_order = mapping_min_folio_order(folio->mapping);
                if (new_order < min_order) {
                        VM_WARN_ONCE(1, "Cannot split mapped folio below min-order: %u",
                                     min_order);
                        ret = -EINVAL;
                        goto out;
                }

                gfp = current_gfp_context(mapping_gfp_mask(mapping) &
                                                        GFP_RECLAIM_MASK);

                if (!filemap_release_folio(folio, gfp)) {
                        ret = -EBUSY;
                        goto out;
                }

                if (uniform_split) {
                        xas_set_order(&xas, folio->index, new_order);
                        xas_split_alloc(&xas, folio, folio_order(folio), gfp);
                        if (xas_error(&xas)) {
                                ret = xas_error(&xas);
                                goto out;
                        }
                }

                anon_vma = NULL;
                i_mmap_lock_read(mapping);

                /*
                 *__split_unmapped_folio() may need to trim off pages beyond
                 * EOF: but on 32-bit, i_size_read() takes an irq-unsafe
                 * seqlock, which cannot be nested inside the page tree lock.
                 * So note end now: i_size itself may be changed at any moment,
                 * but folio lock is good enough to serialize the trimming.
                 */
                end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
                if (shmem_mapping(mapping))
                        end = shmem_fallocend(mapping->host, end);
        }

        /*
         * Racy check if we can split the page, before unmap_folio() will
         * split PMDs
         */
        if (!can_split_folio(folio, 1, &extra_pins)) {
                ret = -EAGAIN;
                goto out_unlock;
        }

        unmap_folio(folio);

        /* block interrupt reentry in xa_lock and spinlock */
        local_irq_disable();
        if (mapping) {
                /*
                 * Check if the folio is present in page cache.
                 * We assume all tail are present too, if folio is there.
                 */
                xas_lock(&xas);
                xas_reset(&xas);
                if (xas_load(&xas) != folio)
                        goto fail;
        }

        /* Prevent deferred_split_scan() touching ->_refcount */
        spin_lock(&ds_queue->split_queue_lock);
        if (folio_ref_freeze(folio, 1 + extra_pins)) {
                if (folio_order(folio) > 1 &&
                    !list_empty(&folio->_deferred_list)) {
                        ds_queue->split_queue_len--;
                        if (folio_test_partially_mapped(folio)) {
                                folio_clear_partially_mapped(folio);
                                mod_mthp_stat(folio_order(folio),
                                              MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
                        }
                        /*
                         * Reinitialize page_deferred_list after removing the
                         * page from the split_queue, otherwise a subsequent
                         * split will see list corruption when checking the
                         * page_deferred_list.
                         */
                        list_del_init(&folio->_deferred_list);
                }
                spin_unlock(&ds_queue->split_queue_lock);
                if (mapping) {
                        int nr = folio_nr_pages(folio);

                        if (folio_test_pmd_mappable(folio) &&
                            new_order < HPAGE_PMD_ORDER) {
                                if (folio_test_swapbacked(folio)) {
                                        __lruvec_stat_mod_folio(folio,
                                                        NR_SHMEM_THPS, -nr);
                                } else {
                                        __lruvec_stat_mod_folio(folio,
                                                        NR_FILE_THPS, -nr);
                                        filemap_nr_thps_dec(mapping);
                                }
                        }
                }

                ret = __split_unmapped_folio(folio, new_order,
                                split_at, lock_at, list, end, &xas, mapping,
                                uniform_split);
        } else {
                spin_unlock(&ds_queue->split_queue_lock);
fail:
                if (mapping)
                        xas_unlock(&xas);
                local_irq_enable();
                remap_page(folio, folio_nr_pages(folio), 0);
                ret = -EAGAIN;
        }

out_unlock:
        if (anon_vma) {
                anon_vma_unlock_write(anon_vma);
                put_anon_vma(anon_vma);
        }
        if (mapping)
                i_mmap_unlock_read(mapping);
out:
        xas_destroy(&xas);
        if (order == HPAGE_PMD_ORDER)
                count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED);
        count_mthp_stat(order, !ret ? MTHP_STAT_SPLIT : MTHP_STAT_SPLIT_FAILED);
        return ret;
}

/*
 * This function splits a large folio into smaller folios of order @new_order.
 * @page can point to any page of the large folio to split. The split operation
 * does not change the position of @page.
 *
 * Prerequisites:
 *
 * 1) The caller must hold a reference on the @page's owning folio, also known
 *    as the large folio.
 *
 * 2) The large folio must be locked.
 *
 * 3) The folio must not be pinned. Any unexpected folio references, including
 *    GUP pins, will result in the folio not getting split; instead, the caller
 *    will receive an -EAGAIN.
 *
 * 4) @new_order > 1, usually. Splitting to order-1 anonymous folios is not
 *    supported for non-file-backed folios, because folio->_deferred_list, which
 *    is used by partially mapped folios, is stored in subpage 2, but an order-1
 *    folio only has subpages 0 and 1. File-backed order-1 folios are supported,
 *    since they do not use _deferred_list.
 *
 * After splitting, the caller's folio reference will be transferred to @page,
 * resulting in a raised refcount of @page after this call. The other pages may
 * be freed if they are not mapped.
 *
 * If @list is null, tail pages will be added to LRU list, otherwise, to @list.
 *
 * Pages in @new_order will inherit the mapping, flags, and so on from the
 * huge page.
 *
 * Returns 0 if the huge page was split successfully.
 *
 * Returns -EAGAIN if the folio has unexpected reference (e.g., GUP) or if
 * the folio was concurrently removed from the page cache.
 *
 * Returns -EBUSY when trying to split the huge zeropage, if the folio is
 * under writeback, if fs-specific folio metadata cannot currently be
 * released, or if some unexpected race happened (e.g., anon VMA disappeared,
 * truncation).
 *
 * Callers should ensure that the order respects the address space mapping
 * min-order if one is set for non-anonymous folios.
 *
 * Returns -EINVAL when trying to split to an order that is incompatible
 * with the folio. Splitting to order 0 is compatible with all folios.
 */
int split_huge_page_to_list_to_order(struct page *page, struct list_head *list,
                                     unsigned int new_order)
{
        struct folio *folio = page_folio(page);

        return __folio_split(folio, new_order, &folio->page, page, list, true);
}

/*
 * folio_split: split a folio at @split_at to a @new_order folio
 * @folio: folio to split
 * @new_order: the order of the new folio
 * @split_at: a page within the new folio
 *
 * return: 0: successful, <0 failed (if -ENOMEM is returned, @folio might be
 * split but not to @new_order, the caller needs to check)
 *
 * It has the same prerequisites and returns as
 * split_huge_page_to_list_to_order().
 *
 * Split a folio at @split_at to a new_order folio, leave the
 * remaining subpages of the original folio as large as possible. For example,
 * in the case of splitting an order-9 folio at its third order-3 subpages to
 * an order-3 folio, there are 2^(9-3)=64 order-3 subpages in the order-9 folio.
 * After the split, there will be a group of folios with different orders and
 * the new folio containing @split_at is marked in bracket:
 * [order-4, {order-3}, order-3, order-5, order-6, order-7, order-8].
 *
 * After split, folio is left locked for caller.
 */
int folio_split(struct folio *folio, unsigned int new_order,
                struct page *split_at, struct list_head *list)
{
        return __folio_split(folio, new_order, split_at, &folio->page, list,
                        false);
}

int min_order_for_split(struct folio *folio)
{
        if (folio_test_anon(folio))
                return 0;

        if (!folio->mapping) {
                if (folio_test_pmd_mappable(folio))
                        count_vm_event(THP_SPLIT_PAGE_FAILED);
                return -EBUSY;
        }

        return mapping_min_folio_order(folio->mapping);
}

int split_folio_to_list(struct folio *folio, struct list_head *list)
{
        int ret = min_order_for_split(folio);

        if (ret < 0)
                return ret;

        return split_huge_page_to_list_to_order(&folio->page, list, ret);
}

/*
 * __folio_unqueue_deferred_split() is not to be called directly:
 * the folio_unqueue_deferred_split() inline wrapper in mm/internal.h
 * limits its calls to those folios which may have a _deferred_list for
 * queueing THP splits, and that list is (racily observed to be) non-empty.
 *
 * It is unsafe to call folio_unqueue_deferred_split() until folio refcount is
 * zero: because even when split_queue_lock is held, a non-empty _deferred_list
 * might be in use on deferred_split_scan()'s unlocked on-stack list.
 *
 * If memory cgroups are enabled, split_queue_lock is in the mem_cgroup: it is
 * therefore important to unqueue deferred split before changing folio memcg.
 */
bool __folio_unqueue_deferred_split(struct folio *folio)
{
        struct deferred_split *ds_queue;
        unsigned long flags;
        bool unqueued = false;

        WARN_ON_ONCE(folio_ref_count(folio));
        WARN_ON_ONCE(!mem_cgroup_disabled() && !folio_memcg(folio));

        ds_queue = get_deferred_split_queue(folio);
        spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
        if (!list_empty(&folio->_deferred_list)) {
                ds_queue->split_queue_len--;
                if (folio_test_partially_mapped(folio)) {
                        folio_clear_partially_mapped(folio);
                        mod_mthp_stat(folio_order(folio),
                                      MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
                }
                list_del_init(&folio->_deferred_list);
                unqueued = true;
        }
        spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);

        return unqueued;        /* useful for debug warnings */
}

/* partially_mapped=false won't clear PG_partially_mapped folio flag */
void deferred_split_folio(struct folio *folio, bool partially_mapped)
{
        struct deferred_split *ds_queue = get_deferred_split_queue(folio);
#ifdef CONFIG_MEMCG
        struct mem_cgroup *memcg = folio_memcg(folio);
#endif
        unsigned long flags;

        /*
         * Order 1 folios have no space for a deferred list, but we also
         * won't waste much memory by not adding them to the deferred list.
         */
        if (folio_order(folio) <= 1)
                return;

        if (!partially_mapped && !split_underused_thp)
                return;

        /*
         * Exclude swapcache: originally to avoid a corrupt deferred split
         * queue. Nowadays that is fully prevented by memcg1_swapout();
         * but if page reclaim is already handling the same folio, it is
         * unnecessary to handle it again in the shrinker, so excluding
         * swapcache here may still be a useful optimization.
         */
        if (folio_test_swapcache(folio))
                return;

        spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
        if (partially_mapped) {
                if (!folio_test_partially_mapped(folio)) {
                        folio_set_partially_mapped(folio);
                        if (folio_test_pmd_mappable(folio))
                                count_vm_event(THP_DEFERRED_SPLIT_PAGE);
                        count_mthp_stat(folio_order(folio), MTHP_STAT_SPLIT_DEFERRED);
                        mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, 1);

                }
        } else {
                /* partially mapped folios cannot become non-partially mapped */
                VM_WARN_ON_FOLIO(folio_test_partially_mapped(folio), folio);
        }
        if (list_empty(&folio->_deferred_list)) {
                list_add_tail(&folio->_deferred_list, &ds_queue->split_queue);
                ds_queue->split_queue_len++;
#ifdef CONFIG_MEMCG
                if (memcg)
                        set_shrinker_bit(memcg, folio_nid(folio),
                                         deferred_split_shrinker->id);
#endif
        }
        spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);
}

static unsigned long deferred_split_count(struct shrinker *shrink,
                struct shrink_control *sc)
{
        struct pglist_data *pgdata = NODE_DATA(sc->nid);
        struct deferred_split *ds_queue = &pgdata->deferred_split_queue;

#ifdef CONFIG_MEMCG
        if (sc->memcg)
                ds_queue = &sc->memcg->deferred_split_queue;
#endif
        return READ_ONCE(ds_queue->split_queue_len);
}

static bool thp_underused(struct folio *folio)
{
        int num_zero_pages = 0, num_filled_pages = 0;
        void *kaddr;
        int i;

        if (khugepaged_max_ptes_none == HPAGE_PMD_NR - 1)
                return false;

        for (i = 0; i < folio_nr_pages(folio); i++) {
                kaddr = kmap_local_folio(folio, i * PAGE_SIZE);
                if (!memchr_inv(kaddr, 0, PAGE_SIZE)) {
                        num_zero_pages++;
                        if (num_zero_pages > khugepaged_max_ptes_none) {
                                kunmap_local(kaddr);
                                return true;
                        }
                } else {
                        /*
                         * Another path for early exit once the number
                         * of non-zero filled pages exceeds threshold.
                         */
                        num_filled_pages++;
                        if (num_filled_pages >= HPAGE_PMD_NR - khugepaged_max_ptes_none) {
                                kunmap_local(kaddr);
                                return false;
                        }
                }
                kunmap_local(kaddr);
        }
        return false;
}

static unsigned long deferred_split_scan(struct shrinker *shrink,
                struct shrink_control *sc)
{
        struct pglist_data *pgdata = NODE_DATA(sc->nid);
        struct deferred_split *ds_queue = &pgdata->deferred_split_queue;
        unsigned long flags;
        LIST_HEAD(list);
        struct folio *folio, *next, *prev = NULL;
        int split = 0, removed = 0;

#ifdef CONFIG_MEMCG
        if (sc->memcg)
                ds_queue = &sc->memcg->deferred_split_queue;
#endif

        spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
        /* Take pin on all head pages to avoid freeing them under us */
        list_for_each_entry_safe(folio, next, &ds_queue->split_queue,
                                                        _deferred_list) {
                if (folio_try_get(folio)) {
                        list_move(&folio->_deferred_list, &list);
                } else {
                        /* We lost race with folio_put() */
                        if (folio_test_partially_mapped(folio)) {
                                folio_clear_partially_mapped(folio);
                                mod_mthp_stat(folio_order(folio),
                                              MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
                        }
                        list_del_init(&folio->_deferred_list);
                        ds_queue->split_queue_len--;
                }
                if (!--sc->nr_to_scan)
                        break;
        }
        spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);

        list_for_each_entry_safe(folio, next, &list, _deferred_list) {
                bool did_split = false;
                bool underused = false;

                if (!folio_test_partially_mapped(folio)) {
                        underused = thp_underused(folio);
                        if (!underused)
                                goto next;
                }
                if (!folio_trylock(folio))
                        goto next;
                if (!split_folio(folio)) {
                        did_split = true;
                        if (underused)
                                count_vm_event(THP_UNDERUSED_SPLIT_PAGE);
                        split++;
                }
                folio_unlock(folio);
next:
                /*
                 * split_folio() removes folio from list on success.
                 * Only add back to the queue if folio is partially mapped.
                 * If thp_underused returns false, or if split_folio fails
                 * in the case it was underused, then consider it used and
                 * don't add it back to split_queue.
                 */
                if (did_split) {
                        ; /* folio already removed from list */
                } else if (!folio_test_partially_mapped(folio)) {
                        list_del_init(&folio->_deferred_list);
                        removed++;
                } else {
                        /*
                         * That unlocked list_del_init() above would be unsafe,
                         * unless its folio is separated from any earlier folios
                         * left on the list (which may be concurrently unqueued)
                         * by one safe folio with refcount still raised.
                         */
                        swap(folio, prev);
                }
                if (folio)
                        folio_put(folio);
        }

        spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
        list_splice_tail(&list, &ds_queue->split_queue);
        ds_queue->split_queue_len -= removed;
        spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);

        if (prev)
                folio_put(prev);

        /*
         * Stop shrinker if we didn't split any page, but the queue is empty.
         * This can happen if pages were freed under us.
         */
        if (!split && list_empty(&ds_queue->split_queue))
                return SHRINK_STOP;
        return split;
}

#ifdef CONFIG_DEBUG_FS
static void split_huge_pages_all(void)
{
        struct zone *zone;
        struct page *page;
        struct folio *folio;
        unsigned long pfn, max_zone_pfn;
        unsigned long total = 0, split = 0;

        pr_debug("Split all THPs\n");
        for_each_zone(zone) {
                if (!managed_zone(zone))
                        continue;
                max_zone_pfn = zone_end_pfn(zone);
                for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {
                        int nr_pages;

                        page = pfn_to_online_page(pfn);
                        if (!page || PageTail(page))
                                continue;
                        folio = page_folio(page);
                        if (!folio_try_get(folio))
                                continue;

                        if (unlikely(page_folio(page) != folio))
                                goto next;

                        if (zone != folio_zone(folio))
                                goto next;

                        if (!folio_test_large(folio)
                                || folio_test_hugetlb(folio)
                                || !folio_test_lru(folio))
                                goto next;

                        total++;
                        folio_lock(folio);
                        nr_pages = folio_nr_pages(folio);
                        if (!split_folio(folio))
                                split++;
                        pfn += nr_pages - 1;
                        folio_unlock(folio);
next:
                        folio_put(folio);
                        cond_resched();
                }
        }

        pr_debug("%lu of %lu THP split\n", split, total);
}

static inline bool vma_not_suitable_for_thp_split(struct vm_area_struct *vma)
{
        return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) ||
                    is_vm_hugetlb_page(vma);
}

static int split_huge_pages_pid(int pid, unsigned long vaddr_start,
                                unsigned long vaddr_end, unsigned int new_order,
                                long in_folio_offset)
{
        int ret = 0;
        struct task_struct *task;
        struct mm_struct *mm;
        unsigned long total = 0, split = 0;
        unsigned long addr;

        vaddr_start &= PAGE_MASK;
        vaddr_end &= PAGE_MASK;

        task = find_get_task_by_vpid(pid);
        if (!task) {
                ret = -ESRCH;
                goto out;
        }

        /* Find the mm_struct */
        mm = get_task_mm(task);
        put_task_struct(task);

        if (!mm) {
                ret = -EINVAL;
                goto out;
        }

        pr_debug("Split huge pages in pid: %d, vaddr: [0x%lx - 0x%lx]\n",
                 pid, vaddr_start, vaddr_end);

        mmap_read_lock(mm);
        /*
         * always increase addr by PAGE_SIZE, since we could have a PTE page
         * table filled with PTE-mapped THPs, each of which is distinct.
         */
        for (addr = vaddr_start; addr < vaddr_end; addr += PAGE_SIZE) {
                struct vm_area_struct *vma = vma_lookup(mm, addr);
                struct folio_walk fw;
                struct folio *folio;
                struct address_space *mapping;
                unsigned int target_order = new_order;

                if (!vma)
                        break;

                /* skip special VMA and hugetlb VMA */
                if (vma_not_suitable_for_thp_split(vma)) {
                        addr = vma->vm_end;
                        continue;
                }

                folio = folio_walk_start(&fw, vma, addr, 0);
                if (!folio)
                        continue;

                if (!is_transparent_hugepage(folio))
                        goto next;

                if (!folio_test_anon(folio)) {
                        mapping = folio->mapping;
                        target_order = max(new_order,
                                           mapping_min_folio_order(mapping));
                }

                if (target_order >= folio_order(folio))
                        goto next;

                total++;
                /*
                 * For folios with private, split_huge_page_to_list_to_order()
                 * will try to drop it before split and then check if the folio
                 * can be split or not. So skip the check here.
                 */
                if (!folio_test_private(folio) &&
                    !can_split_folio(folio, 0, NULL))
                        goto next;

                if (!folio_trylock(folio))
                        goto next;
                folio_get(folio);
                folio_walk_end(&fw, vma);

                if (!folio_test_anon(folio) && folio->mapping != mapping)
                        goto unlock;

                if (in_folio_offset < 0 ||
                    in_folio_offset >= folio_nr_pages(folio)) {
                        if (!split_folio_to_order(folio, target_order))
                                split++;
                } else {
                        struct page *split_at = folio_page(folio,
                                                           in_folio_offset);
                        if (!folio_split(folio, target_order, split_at, NULL))
                                split++;
                }

unlock:

                folio_unlock(folio);
                folio_put(folio);

                cond_resched();
                continue;
next:
                folio_walk_end(&fw, vma);
                cond_resched();
        }
        mmap_read_unlock(mm);
        mmput(mm);

        pr_debug("%lu of %lu THP split\n", split, total);

out:
        return ret;
}

static int split_huge_pages_in_file(const char *file_path, pgoff_t off_start,
                                pgoff_t off_end, unsigned int new_order,
                                long in_folio_offset)
{
        struct filename *file;
        struct file *candidate;
        struct address_space *mapping;
        int ret = -EINVAL;
        pgoff_t index;
        int nr_pages = 1;
        unsigned long total = 0, split = 0;
        unsigned int min_order;
        unsigned int target_order;

        file = getname_kernel(file_path);
        if (IS_ERR(file))
                return ret;

        candidate = file_open_name(file, O_RDONLY, 0);
        if (IS_ERR(candidate))
                goto out;

        pr_debug("split file-backed THPs in file: %s, page offset: [0x%lx - 0x%lx]\n",
                 file_path, off_start, off_end);

        mapping = candidate->f_mapping;
        min_order = mapping_min_folio_order(mapping);
        target_order = max(new_order, min_order);

        for (index = off_start; index < off_end; index += nr_pages) {
                struct folio *folio = filemap_get_folio(mapping, index);

                nr_pages = 1;
                if (IS_ERR(folio))
                        continue;

                if (!folio_test_large(folio))
                        goto next;

                total++;
                nr_pages = folio_nr_pages(folio);

                if (target_order >= folio_order(folio))
                        goto next;

                if (!folio_trylock(folio))
                        goto next;

                if (folio->mapping != mapping)
                        goto unlock;

                if (in_folio_offset < 0 || in_folio_offset >= nr_pages) {
                        if (!split_folio_to_order(folio, target_order))
                                split++;
                } else {
                        struct page *split_at = folio_page(folio,
                                                           in_folio_offset);
                        if (!folio_split(folio, target_order, split_at, NULL))
                                split++;
                }

unlock:
                folio_unlock(folio);
next:
                folio_put(folio);
                cond_resched();
        }

        filp_close(candidate, NULL);
        ret = 0;

        pr_debug("%lu of %lu file-backed THP split\n", split, total);
out:
        putname(file);
        return ret;
}

#define MAX_INPUT_BUF_SZ 255

static ssize_t split_huge_pages_write(struct file *file, const char __user *buf,
                                size_t count, loff_t *ppops)
{
        static DEFINE_MUTEX(split_debug_mutex);
        ssize_t ret;
        /*
         * hold pid, start_vaddr, end_vaddr, new_order or
         * file_path, off_start, off_end, new_order
         */
        char input_buf[MAX_INPUT_BUF_SZ];
        int pid;
        unsigned long vaddr_start, vaddr_end;
        unsigned int new_order = 0;
        long in_folio_offset = -1;

        ret = mutex_lock_interruptible(&split_debug_mutex);
        if (ret)
                return ret;

        ret = -EFAULT;

        memset(input_buf, 0, MAX_INPUT_BUF_SZ);
        if (copy_from_user(input_buf, buf, min_t(size_t, count, MAX_INPUT_BUF_SZ)))
                goto out;

        input_buf[MAX_INPUT_BUF_SZ - 1] = '\0';

        if (input_buf[0] == '/') {
                char *tok;
                char *tok_buf = input_buf;
                char file_path[MAX_INPUT_BUF_SZ];
                pgoff_t off_start = 0, off_end = 0;
                size_t input_len = strlen(input_buf);

                tok = strsep(&tok_buf, ",");
                if (tok && tok_buf) {
                        strscpy(file_path, tok);
                } else {
                        ret = -EINVAL;
                        goto out;
                }

                ret = sscanf(tok_buf, "0x%lx,0x%lx,%d,%ld", &off_start, &off_end,
                                &new_order, &in_folio_offset);
                if (ret != 2 && ret != 3 && ret != 4) {
                        ret = -EINVAL;
                        goto out;
                }
                ret = split_huge_pages_in_file(file_path, off_start, off_end,
                                new_order, in_folio_offset);
                if (!ret)
                        ret = input_len;

                goto out;
        }

        ret = sscanf(input_buf, "%d,0x%lx,0x%lx,%d,%ld", &pid, &vaddr_start,
                        &vaddr_end, &new_order, &in_folio_offset);
        if (ret == 1 && pid == 1) {
                split_huge_pages_all();
                ret = strlen(input_buf);
                goto out;
        } else if (ret != 3 && ret != 4 && ret != 5) {
                ret = -EINVAL;
                goto out;
        }

        ret = split_huge_pages_pid(pid, vaddr_start, vaddr_end, new_order,
                        in_folio_offset);
        if (!ret)
                ret = strlen(input_buf);
out:
        mutex_unlock(&split_debug_mutex);
        return ret;

}

static const struct file_operations split_huge_pages_fops = {
        .owner         = THIS_MODULE,
        .write         = split_huge_pages_write,
};

static int __init split_huge_pages_debugfs(void)
{
        debugfs_create_file("split_huge_pages", 0200, NULL, NULL,
                            &split_huge_pages_fops);
        return 0;
}
late_initcall(split_huge_pages_debugfs);
#endif

#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
                struct page *page)
{
        struct folio *folio = page_folio(page);
        struct vm_area_struct *vma = pvmw->vma;
        struct mm_struct *mm = vma->vm_mm;
        unsigned long address = pvmw->address;
        bool anon_exclusive;
        pmd_t pmdval;
        swp_entry_t entry;
        pmd_t pmdswp;

        if (!(pvmw->pmd && !pvmw->pte))
                return 0;

        flush_cache_range(vma, address, address + HPAGE_PMD_SIZE);
        pmdval = pmdp_invalidate(vma, address, pvmw->pmd);

        /* See folio_try_share_anon_rmap_pmd(): invalidate PMD first. */
        anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(page);
        if (anon_exclusive && folio_try_share_anon_rmap_pmd(folio, page)) {
                set_pmd_at(mm, address, pvmw->pmd, pmdval);
                return -EBUSY;
        }

        if (pmd_dirty(pmdval))
                folio_mark_dirty(folio);
        if (pmd_write(pmdval))
                entry = make_writable_migration_entry(page_to_pfn(page));
        else if (anon_exclusive)
                entry = make_readable_exclusive_migration_entry(page_to_pfn(page));
        else
                entry = make_readable_migration_entry(page_to_pfn(page));
        if (pmd_young(pmdval))
                entry = make_migration_entry_young(entry);
        if (pmd_dirty(pmdval))
                entry = make_migration_entry_dirty(entry);
        pmdswp = swp_entry_to_pmd(entry);
        if (pmd_soft_dirty(pmdval))
                pmdswp = pmd_swp_mksoft_dirty(pmdswp);
        if (pmd_uffd_wp(pmdval))
                pmdswp = pmd_swp_mkuffd_wp(pmdswp);
        set_pmd_at(mm, address, pvmw->pmd, pmdswp);
        folio_remove_rmap_pmd(folio, page, vma);
        folio_put(folio);
        trace_set_migration_pmd(address, pmd_val(pmdswp));

        return 0;
}

void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new)
{
        struct folio *folio = page_folio(new);
        struct vm_area_struct *vma = pvmw->vma;
        struct mm_struct *mm = vma->vm_mm;
        unsigned long address = pvmw->address;
        unsigned long haddr = address & HPAGE_PMD_MASK;
        pmd_t pmde;
        swp_entry_t entry;

        if (!(pvmw->pmd && !pvmw->pte))
                return;

        entry = pmd_to_swp_entry(*pvmw->pmd);
        folio_get(folio);
        pmde = mk_huge_pmd(new, READ_ONCE(vma->vm_page_prot));
        if (pmd_swp_soft_dirty(*pvmw->pmd))
                pmde = pmd_mksoft_dirty(pmde);
        if (is_writable_migration_entry(entry))
                pmde = pmd_mkwrite(pmde, vma);
        if (pmd_swp_uffd_wp(*pvmw->pmd))
                pmde = pmd_mkuffd_wp(pmde);
        if (!is_migration_entry_young(entry))
                pmde = pmd_mkold(pmde);
        /* NOTE: this may contain setting soft-dirty on some archs */
        if (folio_test_dirty(folio) && is_migration_entry_dirty(entry))
                pmde = pmd_mkdirty(pmde);

        if (folio_test_anon(folio)) {
                rmap_t rmap_flags = RMAP_NONE;

                if (!is_readable_migration_entry(entry))
                        rmap_flags |= RMAP_EXCLUSIVE;

                folio_add_anon_rmap_pmd(folio, new, vma, haddr, rmap_flags);
        } else {
                folio_add_file_rmap_pmd(folio, new, vma);
        }
        VM_BUG_ON(pmd_write(pmde) && folio_test_anon(folio) && !PageAnonExclusive(new));
        set_pmd_at(mm, haddr, pvmw->pmd, pmde);

        /* No need to invalidate - it was non-present before */
        update_mmu_cache_pmd(vma, address, pvmw->pmd);
        trace_remove_migration_pmd(address, pmd_val(pmde));
}
#endif























































































































































































































































































































































































































































   29 










   29 
























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
/*
 * proc.c - procfs support for Protocol family CAN core module
 *
 * Copyright (c) 2002-2007 Volkswagen Group Electronic Research
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of Volkswagen nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * Alternatively, provided that this notice is retained in full, this
 * software may be distributed under the terms of the GNU General
 * Public License ("GPL") version 2, in which case the provisions of the
 * GPL apply INSTEAD OF those given above.
 *
 * The provided data structures and external interfaces from this code
 * are not restricted to be used by modules with a GPL compatible license.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
 * DAMAGE.
 *
 */

#include <linux/module.h>
#include <linux/proc_fs.h>
#include <linux/list.h>
#include <linux/rcupdate.h>
#include <linux/if_arp.h>
#include <linux/can/can-ml.h>
#include <linux/can/core.h>

#include "af_can.h"

/*
 * proc filenames for the PF_CAN core
 */

#define CAN_PROC_STATS       "stats"
#define CAN_PROC_RESET_STATS "reset_stats"
#define CAN_PROC_RCVLIST_ALL "rcvlist_all"
#define CAN_PROC_RCVLIST_FIL "rcvlist_fil"
#define CAN_PROC_RCVLIST_INV "rcvlist_inv"
#define CAN_PROC_RCVLIST_SFF "rcvlist_sff"
#define CAN_PROC_RCVLIST_EFF "rcvlist_eff"
#define CAN_PROC_RCVLIST_ERR "rcvlist_err"

static int user_reset;

static const char rx_list_name[][8] = {
        [RX_ERR] = "rx_err",
        [RX_ALL] = "rx_all",
        [RX_FIL] = "rx_fil",
        [RX_INV] = "rx_inv",
};

/*
 * af_can statistics stuff
 */

static void can_init_stats(struct net *net)
{
        struct can_pkg_stats *pkg_stats = net->can.pkg_stats;
        struct can_rcv_lists_stats *rcv_lists_stats = net->can.rcv_lists_stats;
        /*
         * This memset function is called from a timer context (when
         * can_stattimer is active which is the default) OR in a process
         * context (reading the proc_fs when can_stattimer is disabled).
         */
        memset(pkg_stats, 0, sizeof(struct can_pkg_stats));
        pkg_stats->jiffies_init = jiffies;

        rcv_lists_stats->stats_reset++;

        if (user_reset) {
                user_reset = 0;
                rcv_lists_stats->user_reset++;
        }
}

static unsigned long calc_rate(unsigned long oldjif, unsigned long newjif,
                               unsigned long count)
{
        if (oldjif == newjif)
                return 0;

        /* see can_stat_update() - this should NEVER happen! */
        if (count > (ULONG_MAX / HZ)) {
                printk(KERN_ERR "can: calc_rate: count exceeded! %ld\n",
                       count);
                return 99999999;
        }

        return (count * HZ) / (newjif - oldjif);
}

void can_stat_update(struct timer_list *t)
{
        struct net *net = from_timer(net, t, can.stattimer);
        struct can_pkg_stats *pkg_stats = net->can.pkg_stats;
        unsigned long j = jiffies; /* snapshot */

        long rx_frames = atomic_long_read(&pkg_stats->rx_frames);
        long tx_frames = atomic_long_read(&pkg_stats->tx_frames);
        long matches = atomic_long_read(&pkg_stats->matches);
        long rx_frames_delta = atomic_long_read(&pkg_stats->rx_frames_delta);
        long tx_frames_delta = atomic_long_read(&pkg_stats->tx_frames_delta);
        long matches_delta = atomic_long_read(&pkg_stats->matches_delta);

        /* restart counting in timer context on user request */
        if (user_reset)
                can_init_stats(net);

        /* restart counting on jiffies overflow */
        if (j < pkg_stats->jiffies_init)
                can_init_stats(net);

        /* prevent overflow in calc_rate() */
        if (rx_frames > (LONG_MAX / HZ))
                can_init_stats(net);

        /* prevent overflow in calc_rate() */
        if (tx_frames > (LONG_MAX / HZ))
                can_init_stats(net);

        /* matches overflow - very improbable */
        if (matches > (LONG_MAX / 100))
                can_init_stats(net);

        /* calc total values */
        if (rx_frames)
                pkg_stats->total_rx_match_ratio = (matches * 100) / rx_frames;

        pkg_stats->total_tx_rate = calc_rate(pkg_stats->jiffies_init, j,
                                            tx_frames);
        pkg_stats->total_rx_rate = calc_rate(pkg_stats->jiffies_init, j,
                                            rx_frames);

        /* calc current values */
        if (rx_frames_delta)
                pkg_stats->current_rx_match_ratio =
                        (matches_delta * 100) /        rx_frames_delta;

        pkg_stats->current_tx_rate = calc_rate(0, HZ, tx_frames_delta);
        pkg_stats->current_rx_rate = calc_rate(0, HZ, rx_frames_delta);

        /* check / update maximum values */
        if (pkg_stats->max_tx_rate < pkg_stats->current_tx_rate)
                pkg_stats->max_tx_rate = pkg_stats->current_tx_rate;

        if (pkg_stats->max_rx_rate < pkg_stats->current_rx_rate)
                pkg_stats->max_rx_rate = pkg_stats->current_rx_rate;

        if (pkg_stats->max_rx_match_ratio < pkg_stats->current_rx_match_ratio)
                pkg_stats->max_rx_match_ratio = pkg_stats->current_rx_match_ratio;

        /* clear values for 'current rate' calculation */
        atomic_long_set(&pkg_stats->tx_frames_delta, 0);
        atomic_long_set(&pkg_stats->rx_frames_delta, 0);
        atomic_long_set(&pkg_stats->matches_delta, 0);

        /* restart timer (one second) */
        mod_timer(&net->can.stattimer, round_jiffies(jiffies + HZ));
}

/*
 * proc read functions
 */

static void can_print_rcvlist(struct seq_file *m, struct hlist_head *rx_list,
                              struct net_device *dev)
{
        struct receiver *r;

        hlist_for_each_entry_rcu(r, rx_list, list) {
                char *fmt = (r->can_id & CAN_EFF_FLAG)?
                        "   %-5s  %08x  %08x  %pK  %pK  %8ld  %s\n" :
                        "   %-5s     %03x    %08x  %pK  %pK  %8ld  %s\n";

                seq_printf(m, fmt, DNAME(dev), r->can_id, r->mask,
                                r->func, r->data, r->matches, r->ident);
        }
}

static void can_print_recv_banner(struct seq_file *m)
{
        /*
         *                  can1.  00000000  00000000  00000000
         *                 .......          0  tp20
         */
        if (IS_ENABLED(CONFIG_64BIT))
                seq_puts(m, "  device   can_id   can_mask      function          userdata       matches  ident\n");
        else
                seq_puts(m, "  device   can_id   can_mask  function  userdata   matches  ident\n");
}

static int can_stats_proc_show(struct seq_file *m, void *v)
{
        struct net *net = m->private;
        struct can_pkg_stats *pkg_stats = net->can.pkg_stats;
        struct can_rcv_lists_stats *rcv_lists_stats = net->can.rcv_lists_stats;

        seq_putc(m, '\n');
        seq_printf(m, " %8ld transmitted frames (TXF)\n",
                   atomic_long_read(&pkg_stats->tx_frames));
        seq_printf(m, " %8ld received frames (RXF)\n",
                   atomic_long_read(&pkg_stats->rx_frames));
        seq_printf(m, " %8ld matched frames (RXMF)\n",
                   atomic_long_read(&pkg_stats->matches));

        seq_putc(m, '\n');

        if (net->can.stattimer.function == can_stat_update) {
                seq_printf(m, " %8ld %% total match ratio (RXMR)\n",
                                pkg_stats->total_rx_match_ratio);

                seq_printf(m, " %8ld frames/s total tx rate (TXR)\n",
                                pkg_stats->total_tx_rate);
                seq_printf(m, " %8ld frames/s total rx rate (RXR)\n",
                                pkg_stats->total_rx_rate);

                seq_putc(m, '\n');

                seq_printf(m, " %8ld %% current match ratio (CRXMR)\n",
                                pkg_stats->current_rx_match_ratio);

                seq_printf(m, " %8ld frames/s current tx rate (CTXR)\n",
                                pkg_stats->current_tx_rate);
                seq_printf(m, " %8ld frames/s current rx rate (CRXR)\n",
                                pkg_stats->current_rx_rate);

                seq_putc(m, '\n');

                seq_printf(m, " %8ld %% max match ratio (MRXMR)\n",
                                pkg_stats->max_rx_match_ratio);

                seq_printf(m, " %8ld frames/s max tx rate (MTXR)\n",
                                pkg_stats->max_tx_rate);
                seq_printf(m, " %8ld frames/s max rx rate (MRXR)\n",
                                pkg_stats->max_rx_rate);

                seq_putc(m, '\n');
        }

        seq_printf(m, " %8ld current receive list entries (CRCV)\n",
                        rcv_lists_stats->rcv_entries);
        seq_printf(m, " %8ld maximum receive list entries (MRCV)\n",
                        rcv_lists_stats->rcv_entries_max);

        if (rcv_lists_stats->stats_reset)
                seq_printf(m, "\n %8ld statistic resets (STR)\n",
                                rcv_lists_stats->stats_reset);

        if (rcv_lists_stats->user_reset)
                seq_printf(m, " %8ld user statistic resets (USTR)\n",
                                rcv_lists_stats->user_reset);

        seq_putc(m, '\n');
        return 0;
}

static int can_reset_stats_proc_show(struct seq_file *m, void *v)
{
        struct net *net = m->private;
        struct can_rcv_lists_stats *rcv_lists_stats = net->can.rcv_lists_stats;
        struct can_pkg_stats *pkg_stats = net->can.pkg_stats;

        user_reset = 1;

        if (net->can.stattimer.function == can_stat_update) {
                seq_printf(m, "Scheduled statistic reset #%ld.\n",
                                rcv_lists_stats->stats_reset + 1);
        } else {
                if (pkg_stats->jiffies_init != jiffies)
                        can_init_stats(net);

                seq_printf(m, "Performed statistic reset #%ld.\n",
                                rcv_lists_stats->stats_reset);
        }
        return 0;
}

static inline void can_rcvlist_proc_show_one(struct seq_file *m, int idx,
                                             struct net_device *dev,
                                             struct can_dev_rcv_lists *dev_rcv_lists)
{
        if (!hlist_empty(&dev_rcv_lists->rx[idx])) {
                can_print_recv_banner(m);
                can_print_rcvlist(m, &dev_rcv_lists->rx[idx], dev);
        } else
                seq_printf(m, "  (%s: no entry)\n", DNAME(dev));

}

static int can_rcvlist_proc_show(struct seq_file *m, void *v)
{
        /* double cast to prevent GCC warning */
        int idx = (int)(long)pde_data(m->file->f_inode);
        struct net_device *dev;
        struct can_dev_rcv_lists *dev_rcv_lists;
        struct net *net = m->private;

        seq_printf(m, "\nreceive list '%s':\n", rx_list_name[idx]);

        rcu_read_lock();

        /* receive list for 'all' CAN devices (dev == NULL) */
        dev_rcv_lists = net->can.rx_alldev_list;
        can_rcvlist_proc_show_one(m, idx, NULL, dev_rcv_lists);

        /* receive list for registered CAN devices */
        for_each_netdev_rcu(net, dev) {
                struct can_ml_priv *can_ml = can_get_ml_priv(dev);

                if (can_ml)
                        can_rcvlist_proc_show_one(m, idx, dev,
                                                  &can_ml->dev_rcv_lists);
        }

        rcu_read_unlock();

        seq_putc(m, '\n');
        return 0;
}

static inline void can_rcvlist_proc_show_array(struct seq_file *m,
                                               struct net_device *dev,
                                               struct hlist_head *rcv_array,
                                               unsigned int rcv_array_sz)
{
        unsigned int i;
        int all_empty = 1;

        /* check whether at least one list is non-empty */
        for (i = 0; i < rcv_array_sz; i++)
                if (!hlist_empty(&rcv_array[i])) {
                        all_empty = 0;
                        break;
                }

        if (!all_empty) {
                can_print_recv_banner(m);
                for (i = 0; i < rcv_array_sz; i++) {
                        if (!hlist_empty(&rcv_array[i]))
                                can_print_rcvlist(m, &rcv_array[i], dev);
                }
        } else
                seq_printf(m, "  (%s: no entry)\n", DNAME(dev));
}

static int can_rcvlist_sff_proc_show(struct seq_file *m, void *v)
{
        struct net_device *dev;
        struct can_dev_rcv_lists *dev_rcv_lists;
        struct net *net = m->private;

        /* RX_SFF */
        seq_puts(m, "\nreceive list 'rx_sff':\n");

        rcu_read_lock();

        /* sff receive list for 'all' CAN devices (dev == NULL) */
        dev_rcv_lists = net->can.rx_alldev_list;
        can_rcvlist_proc_show_array(m, NULL, dev_rcv_lists->rx_sff,
                                    ARRAY_SIZE(dev_rcv_lists->rx_sff));

        /* sff receive list for registered CAN devices */
        for_each_netdev_rcu(net, dev) {
                struct can_ml_priv *can_ml = can_get_ml_priv(dev);

                if (can_ml) {
                        dev_rcv_lists = &can_ml->dev_rcv_lists;
                        can_rcvlist_proc_show_array(m, dev, dev_rcv_lists->rx_sff,
                                                    ARRAY_SIZE(dev_rcv_lists->rx_sff));
                }
        }

        rcu_read_unlock();

        seq_putc(m, '\n');
        return 0;
}

static int can_rcvlist_eff_proc_show(struct seq_file *m, void *v)
{
        struct net_device *dev;
        struct can_dev_rcv_lists *dev_rcv_lists;
        struct net *net = m->private;

        /* RX_EFF */
        seq_puts(m, "\nreceive list 'rx_eff':\n");

        rcu_read_lock();

        /* eff receive list for 'all' CAN devices (dev == NULL) */
        dev_rcv_lists = net->can.rx_alldev_list;
        can_rcvlist_proc_show_array(m, NULL, dev_rcv_lists->rx_eff,
                                    ARRAY_SIZE(dev_rcv_lists->rx_eff));

        /* eff receive list for registered CAN devices */
        for_each_netdev_rcu(net, dev) {
                struct can_ml_priv *can_ml = can_get_ml_priv(dev);

                if (can_ml) {
                        dev_rcv_lists = &can_ml->dev_rcv_lists;
                        can_rcvlist_proc_show_array(m, dev, dev_rcv_lists->rx_eff,
                                                    ARRAY_SIZE(dev_rcv_lists->rx_eff));
                }
        }

        rcu_read_unlock();

        seq_putc(m, '\n');
        return 0;
}

/*
 * can_init_proc - create main CAN proc directory and procfs entries
 */
void can_init_proc(struct net *net)
{
        /* create /proc/net/can directory */
        net->can.proc_dir = proc_net_mkdir(net, "can", net->proc_net);

        if (!net->can.proc_dir) {
                printk(KERN_INFO "can: failed to create /proc/net/can . "
                           "CONFIG_PROC_FS missing?\n");
                return;
        }

        /* own procfs entries from the AF_CAN core */
        net->can.pde_stats = proc_create_net_single(CAN_PROC_STATS, 0644,
                        net->can.proc_dir, can_stats_proc_show, NULL);
        net->can.pde_reset_stats = proc_create_net_single(CAN_PROC_RESET_STATS,
                        0644, net->can.proc_dir, can_reset_stats_proc_show,
                        NULL);
        net->can.pde_rcvlist_err = proc_create_net_single(CAN_PROC_RCVLIST_ERR,
                        0644, net->can.proc_dir, can_rcvlist_proc_show,
                        (void *)RX_ERR);
        net->can.pde_rcvlist_all = proc_create_net_single(CAN_PROC_RCVLIST_ALL,
                        0644, net->can.proc_dir, can_rcvlist_proc_show,
                        (void *)RX_ALL);
        net->can.pde_rcvlist_fil = proc_create_net_single(CAN_PROC_RCVLIST_FIL,
                        0644, net->can.proc_dir, can_rcvlist_proc_show,
                        (void *)RX_FIL);
        net->can.pde_rcvlist_inv = proc_create_net_single(CAN_PROC_RCVLIST_INV,
                        0644, net->can.proc_dir, can_rcvlist_proc_show,
                        (void *)RX_INV);
        net->can.pde_rcvlist_eff = proc_create_net_single(CAN_PROC_RCVLIST_EFF,
                        0644, net->can.proc_dir, can_rcvlist_eff_proc_show, NULL);
        net->can.pde_rcvlist_sff = proc_create_net_single(CAN_PROC_RCVLIST_SFF,
                        0644, net->can.proc_dir, can_rcvlist_sff_proc_show, NULL);
}

/*
 * can_remove_proc - remove procfs entries and main CAN proc directory
 */
void can_remove_proc(struct net *net)
{
        if (!net->can.proc_dir)
                return;

        if (net->can.pde_stats)
                remove_proc_entry(CAN_PROC_STATS, net->can.proc_dir);

        if (net->can.pde_reset_stats)
                remove_proc_entry(CAN_PROC_RESET_STATS, net->can.proc_dir);

        if (net->can.pde_rcvlist_err)
                remove_proc_entry(CAN_PROC_RCVLIST_ERR, net->can.proc_dir);

        if (net->can.pde_rcvlist_all)
                remove_proc_entry(CAN_PROC_RCVLIST_ALL, net->can.proc_dir);

        if (net->can.pde_rcvlist_fil)
                remove_proc_entry(CAN_PROC_RCVLIST_FIL, net->can.proc_dir);

        if (net->can.pde_rcvlist_inv)
                remove_proc_entry(CAN_PROC_RCVLIST_INV, net->can.proc_dir);

        if (net->can.pde_rcvlist_eff)
                remove_proc_entry(CAN_PROC_RCVLIST_EFF, net->can.proc_dir);

        if (net->can.pde_rcvlist_sff)
                remove_proc_entry(CAN_PROC_RCVLIST_SFF, net->can.proc_dir);

        remove_proc_entry("can", net->proc_net);
}






















   13 
   13 

































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/core/netclassid_cgroup.c        Classid Cgroupfs Handling
 *
 * Authors:        Thomas Graf <tgraf@suug.ch>
 */

#include <linux/slab.h>
#include <linux/cgroup.h>
#include <linux/fdtable.h>
#include <linux/sched/task.h>

#include <net/cls_cgroup.h>
#include <net/sock.h>

static inline struct cgroup_cls_state *css_cls_state(struct cgroup_subsys_state *css)
{
        return css ? container_of(css, struct cgroup_cls_state, css) : NULL;
}

struct cgroup_cls_state *task_cls_state(struct task_struct *p)
{
        return css_cls_state(task_css_check(p, net_cls_cgrp_id,
                                            rcu_read_lock_bh_held()));
}
EXPORT_SYMBOL_GPL(task_cls_state);

static struct cgroup_subsys_state *
cgrp_css_alloc(struct cgroup_subsys_state *parent_css)
{
        struct cgroup_cls_state *cs;

        cs = kzalloc(sizeof(*cs), GFP_KERNEL);
        if (!cs)
                return ERR_PTR(-ENOMEM);

        return &cs->css;
}

static int cgrp_css_online(struct cgroup_subsys_state *css)
{
        struct cgroup_cls_state *cs = css_cls_state(css);
        struct cgroup_cls_state *parent = css_cls_state(css->parent);

        if (parent)
                cs->classid = parent->classid;

        return 0;
}

static void cgrp_css_free(struct cgroup_subsys_state *css)
{
        kfree(css_cls_state(css));
}

/*
 * To avoid freezing of sockets creation for tasks with big number of threads
 * and opened sockets lets release file_lock every 1000 iterated descriptors.
 * New sockets will already have been created with new classid.
 */

struct update_classid_context {
        u32 classid;
        unsigned int batch;
};

#define UPDATE_CLASSID_BATCH 1000

static int update_classid_sock(const void *v, struct file *file, unsigned int n)
{
        struct update_classid_context *ctx = (void *)v;
        struct socket *sock = sock_from_file(file);

        if (sock)
                sock_cgroup_set_classid(&sock->sk->sk_cgrp_data, ctx->classid);
        if (--ctx->batch == 0) {
                ctx->batch = UPDATE_CLASSID_BATCH;
                return n + 1;
        }
        return 0;
}

static void update_classid_task(struct task_struct *p, u32 classid)
{
        struct update_classid_context ctx = {
                .classid = classid,
                .batch = UPDATE_CLASSID_BATCH
        };
        unsigned int fd = 0;

        /* Only update the leader task, when many threads in this task,
         * so it can avoid the useless traversal.
         */
        if (p != p->group_leader)
                return;

        do {
                task_lock(p);
                fd = iterate_fd(p->files, fd, update_classid_sock, &ctx);
                task_unlock(p);
                cond_resched();
        } while (fd);
}

static void cgrp_attach(struct cgroup_taskset *tset)
{
        struct cgroup_subsys_state *css;
        struct task_struct *p;

        cgroup_taskset_for_each(p, css, tset) {
                update_classid_task(p, css_cls_state(css)->classid);
        }
}

static u64 read_classid(struct cgroup_subsys_state *css, struct cftype *cft)
{
        return css_cls_state(css)->classid;
}

static int write_classid(struct cgroup_subsys_state *css, struct cftype *cft,
                         u64 value)
{
        struct cgroup_cls_state *cs = css_cls_state(css);
        struct css_task_iter it;
        struct task_struct *p;

        cs->classid = (u32)value;

        css_task_iter_start(css, 0, &it);
        while ((p = css_task_iter_next(&it)))
                update_classid_task(p, cs->classid);
        css_task_iter_end(&it);

        return 0;
}

static struct cftype ss_files[] = {
        {
                .name                = "classid",
                .read_u64        = read_classid,
                .write_u64        = write_classid,
        },
        { }        /* terminate */
};

struct cgroup_subsys net_cls_cgrp_subsys = {
        .css_alloc                = cgrp_css_alloc,
        .css_online                = cgrp_css_online,
        .css_free                = cgrp_css_free,
        .attach                        = cgrp_attach,
        .legacy_cftypes                = ss_files,
};

























































































 1028 


































































































































    7 





    7 

    5 




    7 



    7 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
// SPDX-License-Identifier: GPL-2.0
/* Copyright 2011-2014 Autronica Fire and Security AS
 *
 * Author(s):
 *        2011-2014 Arvid Brodin, arvid.brodin@alten.se
 *
 * Frame handler other utility functions for HSR and PRP.
 */

#include "hsr_slave.h"
#include <linux/etherdevice.h>
#include <linux/if_arp.h>
#include <linux/if_vlan.h>
#include "hsr_main.h"
#include "hsr_device.h"
#include "hsr_forward.h"
#include "hsr_framereg.h"

bool hsr_invalid_dan_ingress_frame(__be16 protocol)
{
        return (protocol != htons(ETH_P_PRP) && protocol != htons(ETH_P_HSR));
}

static rx_handler_result_t hsr_handle_frame(struct sk_buff **pskb)
{
        struct sk_buff *skb = *pskb;
        struct hsr_port *port;
        struct hsr_priv *hsr;
        __be16 protocol;

        /* Packets from dev_loopback_xmit() do not have L2 header, bail out */
        if (unlikely(skb->pkt_type == PACKET_LOOPBACK))
                return RX_HANDLER_PASS;

        if (!skb_mac_header_was_set(skb)) {
                WARN_ONCE(1, "%s: skb invalid", __func__);
                return RX_HANDLER_PASS;
        }

        port = hsr_port_get_rcu(skb->dev);
        if (!port)
                goto finish_pass;
        hsr = port->hsr;

        if (hsr_addr_is_self(port->hsr, eth_hdr(skb)->h_source)) {
                /* Directly kill frames sent by ourselves */
                kfree_skb(skb);
                goto finish_consume;
        }

        /* For HSR, only tagged frames are expected (unless the device offloads
         * HSR tag removal), but for PRP there could be non tagged frames as
         * well from Single attached nodes (SANs).
         */
        protocol = eth_hdr(skb)->h_proto;

        if (!(port->dev->features & NETIF_F_HW_HSR_TAG_RM) &&
            port->type != HSR_PT_INTERLINK &&
            hsr->proto_ops->invalid_dan_ingress_frame &&
            hsr->proto_ops->invalid_dan_ingress_frame(protocol))
                goto finish_pass;

        skb_push(skb, ETH_HLEN);
        skb_reset_mac_header(skb);
        if ((!hsr->prot_version && protocol == htons(ETH_P_PRP)) ||
            protocol == htons(ETH_P_HSR))
                skb_set_network_header(skb, ETH_HLEN + HSR_HLEN);
        skb_reset_mac_len(skb);

        /* Only the frames received over the interlink port will assign a
         * sequence number and require synchronisation vs other sender.
         */
        if (port->type == HSR_PT_INTERLINK) {
                spin_lock_bh(&hsr->seqnr_lock);
                hsr_forward_skb(skb, port);
                spin_unlock_bh(&hsr->seqnr_lock);
        } else {
                hsr_forward_skb(skb, port);
        }

finish_consume:
        return RX_HANDLER_CONSUMED;

finish_pass:
        return RX_HANDLER_PASS;
}

bool hsr_port_exists(const struct net_device *dev)
{
        return rcu_access_pointer(dev->rx_handler) == hsr_handle_frame;
}

static int hsr_check_dev_ok(struct net_device *dev,
                            struct netlink_ext_ack *extack)
{
        /* Don't allow HSR on non-ethernet like devices */
        if ((dev->flags & IFF_LOOPBACK) || dev->type != ARPHRD_ETHER ||
            dev->addr_len != ETH_ALEN) {
                NL_SET_ERR_MSG_MOD(extack, "Cannot use loopback or non-ethernet device as HSR slave.");
                return -EINVAL;
        }

        /* Don't allow enslaving hsr devices */
        if (is_hsr_master(dev)) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Cannot create trees of HSR devices.");
                return -EINVAL;
        }

        if (hsr_port_exists(dev)) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "This device is already a HSR slave.");
                return -EINVAL;
        }

        if (is_vlan_dev(dev)) {
                NL_SET_ERR_MSG_MOD(extack, "HSR on top of VLAN is not yet supported in this driver.");
                return -EINVAL;
        }

        if (dev->priv_flags & IFF_DONT_BRIDGE) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "This device does not support bridging.");
                return -EOPNOTSUPP;
        }

        /* HSR over bonded devices has not been tested, but I'm not sure it
         * won't work...
         */

        return 0;
}

/* Setup device to be added to the HSR bridge. */
static int hsr_portdev_setup(struct hsr_priv *hsr, struct net_device *dev,
                             struct hsr_port *port,
                             struct netlink_ext_ack *extack)

{
        struct net_device *hsr_dev;
        struct hsr_port *master;
        int res;

        /* Don't use promiscuous mode for offload since L2 frame forward
         * happens at the offloaded hardware.
         */
        if (!port->hsr->fwd_offloaded) {
                res = dev_set_promiscuity(dev, 1);
                if (res)
                        return res;
        }

        master = hsr_port_get_hsr(hsr, HSR_PT_MASTER);
        hsr_dev = master->dev;

        res = netdev_upper_dev_link(dev, hsr_dev, extack);
        if (res)
                goto fail_upper_dev_link;

        res = netdev_rx_handler_register(dev, hsr_handle_frame, port);
        if (res)
                goto fail_rx_handler;
        dev_disable_lro(dev);

        return 0;

fail_rx_handler:
        netdev_upper_dev_unlink(dev, hsr_dev);
fail_upper_dev_link:
        if (!port->hsr->fwd_offloaded)
                dev_set_promiscuity(dev, -1);

        return res;
}

int hsr_add_port(struct hsr_priv *hsr, struct net_device *dev,
                 enum hsr_port_type type, struct netlink_ext_ack *extack)
{
        struct hsr_port *port, *master;
        int res;

        if (type != HSR_PT_MASTER) {
                res = hsr_check_dev_ok(dev, extack);
                if (res)
                        return res;
        }

        port = hsr_port_get_hsr(hsr, type);
        if (port)
                return -EBUSY;        /* This port already exists */

        port = kzalloc(sizeof(*port), GFP_KERNEL);
        if (!port)
                return -ENOMEM;

        port->hsr = hsr;
        port->dev = dev;
        port->type = type;

        if (type != HSR_PT_MASTER) {
                res = hsr_portdev_setup(hsr, dev, port, extack);
                if (res)
                        goto fail_dev_setup;
        }

        list_add_tail_rcu(&port->port_list, &hsr->ports);

        master = hsr_port_get_hsr(hsr, HSR_PT_MASTER);
        netdev_update_features(master->dev);
        dev_set_mtu(master->dev, hsr_get_max_mtu(hsr));

        return 0;

fail_dev_setup:
        kfree(port);
        return res;
}

void hsr_del_port(struct hsr_port *port)
{
        struct hsr_priv *hsr;
        struct hsr_port *master;

        hsr = port->hsr;
        master = hsr_port_get_hsr(hsr, HSR_PT_MASTER);
        list_del_rcu(&port->port_list);

        if (port != master) {
                netdev_update_features(master->dev);
                dev_set_mtu(master->dev, hsr_get_max_mtu(hsr));
                netdev_rx_handler_unregister(port->dev);
                if (!port->hsr->fwd_offloaded)
                        dev_set_promiscuity(port->dev, -1);
                netdev_upper_dev_unlink(port->dev, master->dev);
        }

        kfree_rcu(port, rcu);
}






























































































































   10 






















   10 

















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __FIRMWARE_LOADER_H
#define __FIRMWARE_LOADER_H

#include <linux/bitops.h>
#include <linux/firmware.h>
#include <linux/types.h>
#include <linux/kref.h>
#include <linux/list.h>
#include <linux/completion.h>

/**
 * enum fw_opt - options to control firmware loading behaviour
 *
 * @FW_OPT_UEVENT: Enables the fallback mechanism to send a kobject uevent
 *        when the firmware is not found. Userspace is in charge to load the
 *        firmware using the sysfs loading facility.
 * @FW_OPT_NOWAIT: Used to describe the firmware request is asynchronous.
 * @FW_OPT_USERHELPER: Enable the fallback mechanism, in case the direct
 *        filesystem lookup fails at finding the firmware.  For details refer to
 *        firmware_fallback_sysfs().
 * @FW_OPT_NO_WARN: Quiet, avoid printing warning messages.
 * @FW_OPT_NOCACHE: Disables firmware caching. Firmware caching is used to
 *        cache the firmware upon suspend, so that upon resume races against the
 *        firmware file lookup on storage is avoided. Used for calls where the
 *        file may be too big, or where the driver takes charge of its own
 *        firmware caching mechanism.
 * @FW_OPT_NOFALLBACK_SYSFS: Disable the sysfs fallback mechanism. Takes
 *        precedence over &FW_OPT_UEVENT and &FW_OPT_USERHELPER.
 * @FW_OPT_FALLBACK_PLATFORM: Enable fallback to device fw copy embedded in
 *        the platform's main firmware. If both this fallback and the sysfs
 *      fallback are enabled, then this fallback will be tried first.
 * @FW_OPT_PARTIAL: Allow partial read of firmware instead of needing to read
 *        entire file.
 */
enum fw_opt {
        FW_OPT_UEVENT                        = BIT(0),
        FW_OPT_NOWAIT                        = BIT(1),
        FW_OPT_USERHELPER                = BIT(2),
        FW_OPT_NO_WARN                        = BIT(3),
        FW_OPT_NOCACHE                        = BIT(4),
        FW_OPT_NOFALLBACK_SYSFS                = BIT(5),
        FW_OPT_FALLBACK_PLATFORM        = BIT(6),
        FW_OPT_PARTIAL                        = BIT(7),
};

enum fw_status {
        FW_STATUS_UNKNOWN,
        FW_STATUS_LOADING,
        FW_STATUS_DONE,
        FW_STATUS_ABORTED,
};

/*
 * Concurrent request_firmware() for the same firmware need to be
 * serialized.  struct fw_state is simple state machine which hold the
 * state of the firmware loading.
 */
struct fw_state {
        struct completion completion;
        enum fw_status status;
};

struct fw_priv {
        struct kref ref;
        struct list_head list;
        struct firmware_cache *fwc;
        struct fw_state fw_st;
        void *data;
        size_t size;
        size_t allocated_size;
        size_t offset;
        u32 opt_flags;
#ifdef CONFIG_FW_LOADER_PAGED_BUF
        bool is_paged_buf;
        struct page **pages;
        int nr_pages;
        int page_array_size;
#endif
#ifdef CONFIG_FW_LOADER_USER_HELPER
        bool need_uevent;
        struct list_head pending_list;
#endif
        const char *fw_name;
};

extern struct mutex fw_lock;
extern struct firmware_cache fw_cache;
extern bool fw_load_abort_all;

static inline bool __fw_state_check(struct fw_priv *fw_priv,
                                    enum fw_status status)
{
        struct fw_state *fw_st = &fw_priv->fw_st;

        return fw_st->status == status;
}

static inline int __fw_state_wait_common(struct fw_priv *fw_priv, long timeout)
{
        struct fw_state *fw_st = &fw_priv->fw_st;
        long ret;

        ret = wait_for_completion_killable_timeout(&fw_st->completion, timeout);
        if (ret != 0 && fw_st->status == FW_STATUS_ABORTED)
                return -ENOENT;
        if (!ret)
                return -ETIMEDOUT;

        return ret < 0 ? ret : 0;
}

static inline void __fw_state_set(struct fw_priv *fw_priv,
                                  enum fw_status status)
{
        struct fw_state *fw_st = &fw_priv->fw_st;

        WRITE_ONCE(fw_st->status, status);

        if (status == FW_STATUS_DONE || status == FW_STATUS_ABORTED) {
#ifdef CONFIG_FW_LOADER_USER_HELPER
                /*
                 * Doing this here ensures that the fw_priv is deleted from
                 * the pending list in all abort/done paths.
                 */
                list_del_init(&fw_priv->pending_list);
#endif
                complete_all(&fw_st->completion);
        }
}

static inline void fw_state_aborted(struct fw_priv *fw_priv)
{
        __fw_state_set(fw_priv, FW_STATUS_ABORTED);
}

static inline bool fw_state_is_aborted(struct fw_priv *fw_priv)
{
        return __fw_state_check(fw_priv, FW_STATUS_ABORTED);
}

static inline void fw_state_start(struct fw_priv *fw_priv)
{
        __fw_state_set(fw_priv, FW_STATUS_LOADING);
}

static inline void fw_state_done(struct fw_priv *fw_priv)
{
        __fw_state_set(fw_priv, FW_STATUS_DONE);
}

static inline bool fw_state_is_done(struct fw_priv *fw_priv)
{
        return __fw_state_check(fw_priv, FW_STATUS_DONE);
}

static inline bool fw_state_is_loading(struct fw_priv *fw_priv)
{
        return __fw_state_check(fw_priv, FW_STATUS_LOADING);
}

int alloc_lookup_fw_priv(const char *fw_name, struct firmware_cache *fwc,
                         struct fw_priv **fw_priv, void *dbuf, size_t size,
                         size_t offset, u32 opt_flags);
int assign_fw(struct firmware *fw, struct device *device);
void free_fw_priv(struct fw_priv *fw_priv);
void fw_state_init(struct fw_priv *fw_priv);

#ifdef CONFIG_FW_LOADER
bool firmware_is_builtin(const struct firmware *fw);
bool firmware_request_builtin_buf(struct firmware *fw, const char *name,
                                  void *buf, size_t size);
#else /* module case */
static inline bool firmware_is_builtin(const struct firmware *fw)
{
        return false;
}
static inline bool firmware_request_builtin_buf(struct firmware *fw,
                                                const char *name,
                                                void *buf, size_t size)
{
        return false;
}
#endif

#ifdef CONFIG_FW_LOADER_PAGED_BUF
void fw_free_paged_buf(struct fw_priv *fw_priv);
int fw_grow_paged_buf(struct fw_priv *fw_priv, int pages_needed);
int fw_map_paged_buf(struct fw_priv *fw_priv);
bool fw_is_paged_buf(struct fw_priv *fw_priv);
#else
static inline void fw_free_paged_buf(struct fw_priv *fw_priv) {}
static inline int fw_grow_paged_buf(struct fw_priv *fw_priv, int pages_needed) { return -ENXIO; }
static inline int fw_map_paged_buf(struct fw_priv *fw_priv) { return -ENXIO; }
static inline bool fw_is_paged_buf(struct fw_priv *fw_priv) { return false; }
#endif

#endif /* __FIRMWARE_LOADER_H */
























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * AppArmor security module
 *
 * This file contains AppArmor network mediation definitions.
 *
 * Copyright (C) 1998-2008 Novell/SUSE
 * Copyright 2009-2017 Canonical Ltd.
 */

#ifndef __AA_NET_H
#define __AA_NET_H

#include <net/sock.h>
#include <linux/path.h>

#include "apparmorfs.h"
#include "label.h"
#include "perms.h"
#include "policy.h"

#define AA_MAY_SEND                AA_MAY_WRITE
#define AA_MAY_RECEIVE                AA_MAY_READ

#define AA_MAY_SHUTDOWN                AA_MAY_DELETE

#define AA_MAY_CONNECT                AA_MAY_OPEN
#define AA_MAY_ACCEPT                0x00100000

#define AA_MAY_BIND                0x00200000
#define AA_MAY_LISTEN                0x00400000

#define AA_MAY_SETOPT                0x01000000
#define AA_MAY_GETOPT                0x02000000

#define NET_PERMS_MASK (AA_MAY_SEND | AA_MAY_RECEIVE | AA_MAY_CREATE |    \
                        AA_MAY_SHUTDOWN | AA_MAY_BIND | AA_MAY_LISTEN |          \
                        AA_MAY_CONNECT | AA_MAY_ACCEPT | AA_MAY_SETATTR | \
                        AA_MAY_GETATTR | AA_MAY_SETOPT | AA_MAY_GETOPT)

#define NET_FS_PERMS (AA_MAY_SEND | AA_MAY_RECEIVE | AA_MAY_CREATE |        \
                      AA_MAY_SHUTDOWN | AA_MAY_CONNECT | AA_MAY_RENAME |\
                      AA_MAY_SETATTR | AA_MAY_GETATTR | AA_MAY_CHMOD |        \
                      AA_MAY_CHOWN | AA_MAY_CHGRP | AA_MAY_LOCK |        \
                      AA_MAY_MPROT)

#define NET_PEER_MASK (AA_MAY_SEND | AA_MAY_RECEIVE | AA_MAY_CONNECT |        \
                       AA_MAY_ACCEPT)
struct aa_sk_ctx {
        struct aa_label *label;
        struct aa_label *peer;
};

static inline struct aa_sk_ctx *aa_sock(const struct sock *sk)
{
        return sk->sk_security + apparmor_blob_sizes.lbs_sock;
}

#define DEFINE_AUDIT_NET(NAME, OP, SK, F, T, P)                                  \
        struct lsm_network_audit NAME ## _net = { .sk = (SK),                  \
                                                  .family = (F)};          \
        DEFINE_AUDIT_DATA(NAME,                                                  \
                          ((SK) && (F) != AF_UNIX) ? LSM_AUDIT_DATA_NET : \
                                                     LSM_AUDIT_DATA_NONE, \
                                                     AA_CLASS_NET,        \
                          OP);                                                  \
        NAME.common.u.net = &(NAME ## _net);                                  \
        NAME.net.type = (T);                                                  \
        NAME.net.protocol = (P)

#define DEFINE_AUDIT_SK(NAME, OP, SK)                                        \
        DEFINE_AUDIT_NET(NAME, OP, SK, (SK)->sk_family, (SK)->sk_type,        \
                         (SK)->sk_protocol)


#define af_select(FAMILY, FN, DEF_FN)                \
({                                                \
        int __e;                                \
        switch ((FAMILY)) {                        \
        default:                                \
                __e = DEF_FN;                        \
        }                                        \
        __e;                                        \
})

struct aa_secmark {
        u8 audit;
        u8 deny;
        u32 secid;
        char *label;
};

extern struct aa_sfs_entry aa_sfs_entry_network[];

void audit_net_cb(struct audit_buffer *ab, void *va);
int aa_profile_af_perm(struct aa_profile *profile,
                       struct apparmor_audit_data *ad,
                       u32 request, u16 family, int type);
int aa_af_perm(const struct cred *subj_cred, struct aa_label *label,
               const char *op, u32 request, u16 family,
               int type, int protocol);
static inline int aa_profile_af_sk_perm(struct aa_profile *profile,
                                        struct apparmor_audit_data *ad,
                                        u32 request,
                                        struct sock *sk)
{
        return aa_profile_af_perm(profile, ad, request, sk->sk_family,
                                  sk->sk_type);
}
int aa_sk_perm(const char *op, u32 request, struct sock *sk);

int aa_sock_file_perm(const struct cred *subj_cred, struct aa_label *label,
                      const char *op, u32 request,
                      struct socket *sock);

int apparmor_secmark_check(struct aa_label *label, char *op, u32 request,
                           u32 secid, const struct sock *sk);

#endif /* __AA_NET_H */


















   24 



    6 


    3 










    4 




    1 



   13 








    1 


   12 


   20 








  509 



  481 






   29 











1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/netlink.h>
#include <linux/nospec.h>
#include <linux/rtnetlink.h>
#include <linux/types.h>
#include <net/ip.h>
#include <net/net_namespace.h>
#include <net/tcp.h>

static int ip_metrics_convert(struct nlattr *fc_mx,
                              int fc_mx_len, u32 *metrics,
                              struct netlink_ext_ack *extack)
{
        bool ecn_ca = false;
        struct nlattr *nla;
        int remaining;

        nla_for_each_attr(nla, fc_mx, fc_mx_len, remaining) {
                int type = nla_type(nla);
                u32 val;

                if (!type)
                        continue;
                if (type > RTAX_MAX) {
                        NL_SET_ERR_MSG(extack, "Invalid metric type");
                        return -EINVAL;
                }

                type = array_index_nospec(type, RTAX_MAX + 1);
                if (type == RTAX_CC_ALGO) {
                        char tmp[TCP_CA_NAME_MAX];

                        nla_strscpy(tmp, nla, sizeof(tmp));
                        val = tcp_ca_get_key_by_name(tmp, &ecn_ca);
                        if (val == TCP_CA_UNSPEC) {
                                NL_SET_ERR_MSG(extack, "Unknown tcp congestion algorithm");
                                return -EINVAL;
                        }
                } else {
                        if (nla_len(nla) != sizeof(u32)) {
                                NL_SET_ERR_MSG_ATTR(extack, nla,
                                                    "Invalid attribute in metrics");
                                return -EINVAL;
                        }
                        val = nla_get_u32(nla);
                }
                if (type == RTAX_ADVMSS && val > 65535 - 40)
                        val = 65535 - 40;
                if (type == RTAX_MTU && val > 65535 - 15)
                        val = 65535 - 15;
                if (type == RTAX_HOPLIMIT && val > 255)
                        val = 255;
                if (type == RTAX_FEATURES && (val & ~RTAX_FEATURE_MASK)) {
                        NL_SET_ERR_MSG(extack, "Unknown flag set in feature mask in metrics attribute");
                        return -EINVAL;
                }
                metrics[type - 1] = val;
        }

        if (ecn_ca)
                metrics[RTAX_FEATURES - 1] |= DST_FEATURE_ECN_CA;

        return 0;
}

struct dst_metrics *ip_fib_metrics_init(struct nlattr *fc_mx,
                                        int fc_mx_len,
                                        struct netlink_ext_ack *extack)
{
        struct dst_metrics *fib_metrics;
        int err;

        if (!fc_mx)
                return (struct dst_metrics *)&dst_default_metrics;

        fib_metrics = kzalloc(sizeof(*fib_metrics), GFP_KERNEL);
        if (unlikely(!fib_metrics))
                return ERR_PTR(-ENOMEM);

        err = ip_metrics_convert(fc_mx, fc_mx_len, fib_metrics->metrics,
                                 extack);
        if (!err) {
                refcount_set(&fib_metrics->refcnt, 1);
        } else {
                kfree(fib_metrics);
                fib_metrics = ERR_PTR(err);
        }

        return fib_metrics;
}
EXPORT_SYMBOL_GPL(ip_fib_metrics_init);
























































































































































































































 1024 




    7 
    7 

























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2007-2012 Nicira, Inc.
 */

#include <linux/if_vlan.h>
#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/ethtool.h>
#include <linux/skbuff.h>

#include <net/dst.h>
#include <net/xfrm.h>
#include <net/rtnetlink.h>

#include "datapath.h"
#include "vport-internal_dev.h"
#include "vport-netdev.h"

struct internal_dev {
        struct vport *vport;
};

static struct vport_ops ovs_internal_vport_ops;

static struct internal_dev *internal_dev_priv(struct net_device *netdev)
{
        return netdev_priv(netdev);
}

/* Called with rcu_read_lock_bh. */
static netdev_tx_t
internal_dev_xmit(struct sk_buff *skb, struct net_device *netdev)
{
        int len, err;

        /* store len value because skb can be freed inside ovs_vport_receive() */
        len = skb->len;

        rcu_read_lock();
        err = ovs_vport_receive(internal_dev_priv(netdev)->vport, skb, NULL);
        rcu_read_unlock();

        if (likely(!err))
                dev_sw_netstats_tx_add(netdev, 1, len);
        else
                netdev->stats.tx_errors++;

        return NETDEV_TX_OK;
}

static int internal_dev_open(struct net_device *netdev)
{
        netif_start_queue(netdev);
        return 0;
}

static int internal_dev_stop(struct net_device *netdev)
{
        netif_stop_queue(netdev);
        return 0;
}

static void internal_dev_getinfo(struct net_device *netdev,
                                 struct ethtool_drvinfo *info)
{
        strscpy(info->driver, "openvswitch", sizeof(info->driver));
}

static const struct ethtool_ops internal_dev_ethtool_ops = {
        .get_drvinfo        = internal_dev_getinfo,
        .get_link        = ethtool_op_get_link,
};

static void internal_dev_destructor(struct net_device *dev)
{
        struct vport *vport = ovs_internal_dev_get_vport(dev);

        ovs_vport_free(vport);
}

static const struct net_device_ops internal_dev_netdev_ops = {
        .ndo_open = internal_dev_open,
        .ndo_stop = internal_dev_stop,
        .ndo_start_xmit = internal_dev_xmit,
        .ndo_set_mac_address = eth_mac_addr,
};

static struct rtnl_link_ops internal_dev_link_ops __read_mostly = {
        .kind = "openvswitch",
};

static void do_setup(struct net_device *netdev)
{
        ether_setup(netdev);

        netdev->max_mtu = ETH_MAX_MTU;

        netdev->netdev_ops = &internal_dev_netdev_ops;

        netdev->priv_flags &= ~IFF_TX_SKB_SHARING;
        netdev->priv_flags |= IFF_LIVE_ADDR_CHANGE | IFF_OPENVSWITCH |
                              IFF_NO_QUEUE;
        netdev->lltx = true;
        netdev->needs_free_netdev = true;
        netdev->priv_destructor = NULL;
        netdev->ethtool_ops = &internal_dev_ethtool_ops;
        netdev->rtnl_link_ops = &internal_dev_link_ops;

        netdev->features = NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HIGHDMA |
                           NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE |
                           NETIF_F_GSO_ENCAP_ALL;

        netdev->vlan_features = netdev->features;
        netdev->hw_enc_features = netdev->features;
        netdev->features |= NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_STAG_TX;
        netdev->hw_features = netdev->features;

        eth_hw_addr_random(netdev);
}

static struct vport *internal_dev_create(const struct vport_parms *parms)
{
        struct vport *vport;
        struct internal_dev *internal_dev;
        struct net_device *dev;
        int err;

        vport = ovs_vport_alloc(0, &ovs_internal_vport_ops, parms);
        if (IS_ERR(vport)) {
                err = PTR_ERR(vport);
                goto error;
        }

        dev = alloc_netdev(sizeof(struct internal_dev),
                           parms->name, NET_NAME_USER, do_setup);
        vport->dev = dev;
        if (!vport->dev) {
                err = -ENOMEM;
                goto error_free_vport;
        }
        dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;

        dev_net_set(vport->dev, ovs_dp_get_net(vport->dp));
        dev->ifindex = parms->desired_ifindex;
        internal_dev = internal_dev_priv(vport->dev);
        internal_dev->vport = vport;

        /* Restrict bridge port to current netns. */
        if (vport->port_no == OVSP_LOCAL)
                vport->dev->netns_immutable = true;

        rtnl_lock();
        err = register_netdevice(vport->dev);
        if (err)
                goto error_unlock;
        vport->dev->priv_destructor = internal_dev_destructor;

        dev_set_promiscuity(vport->dev, 1);
        rtnl_unlock();
        netif_start_queue(vport->dev);

        return vport;

error_unlock:
        rtnl_unlock();
        free_netdev(dev);
error_free_vport:
        ovs_vport_free(vport);
error:
        return ERR_PTR(err);
}

static void internal_dev_destroy(struct vport *vport)
{
        netif_stop_queue(vport->dev);
        rtnl_lock();
        dev_set_promiscuity(vport->dev, -1);

        /* unregister_netdevice() waits for an RCU grace period. */
        unregister_netdevice(vport->dev);
        rtnl_unlock();
}

static int internal_dev_recv(struct sk_buff *skb)
{
        struct net_device *netdev = skb->dev;

        if (unlikely(!(netdev->flags & IFF_UP))) {
                kfree_skb(skb);
                netdev->stats.rx_dropped++;
                return NETDEV_TX_OK;
        }

        skb_dst_drop(skb);
        nf_reset_ct(skb);

        skb->pkt_type = PACKET_HOST;
        skb->protocol = eth_type_trans(skb, netdev);
        skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
        dev_sw_netstats_rx_add(netdev, skb->len);

        netif_rx(skb);
        return NETDEV_TX_OK;
}

static struct vport_ops ovs_internal_vport_ops = {
        .type                = OVS_VPORT_TYPE_INTERNAL,
        .create                = internal_dev_create,
        .destroy        = internal_dev_destroy,
        .send                = internal_dev_recv,
};

int ovs_is_internal_dev(const struct net_device *netdev)
{
        return netdev->netdev_ops == &internal_dev_netdev_ops;
}

struct vport *ovs_internal_dev_get_vport(struct net_device *netdev)
{
        if (!ovs_is_internal_dev(netdev))
                return NULL;

        return internal_dev_priv(netdev)->vport;
}

int ovs_internal_dev_rtnl_link_register(void)
{
        int err;

        err = rtnl_link_register(&internal_dev_link_ops);
        if (err < 0)
                return err;

        err = ovs_vport_ops_register(&ovs_internal_vport_ops);
        if (err < 0)
                rtnl_link_unregister(&internal_dev_link_ops);

        return err;
}

void ovs_internal_dev_rtnl_link_unregister(void)
{
        ovs_vport_ops_unregister(&ovs_internal_vport_ops);
        rtnl_link_unregister(&internal_dev_link_ops);
}



































































































   31 










   32 







    2 












    2 






















   41 


   40 


   40 



   39 










































  308 















  309 
  308 
  309 


  307 
  308 
  306 
  308 






  307 



  308 
















































  309 
























  291 

   21 



















  292 






  292 



   21 






































































































































































































































   10 

















   10 
   10 
   10 












   10 






   10 
































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
// SPDX-License-Identifier: GPL-2.0-or-later
/* Task credentials management - see Documentation/security/credentials.rst
 *
 * Copyright (C) 2008 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#define pr_fmt(fmt) "CRED: " fmt

#include <linux/export.h>
#include <linux/cred.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/sched/coredump.h>
#include <linux/key.h>
#include <linux/keyctl.h>
#include <linux/init_task.h>
#include <linux/security.h>
#include <linux/binfmts.h>
#include <linux/cn_proc.h>
#include <linux/uidgid.h>

#if 0
#define kdebug(FMT, ...)                                                \
        printk("[%-5.5s%5u] " FMT "\n",                                        \
               current->comm, current->pid, ##__VA_ARGS__)
#else
#define kdebug(FMT, ...)                                                \
do {                                                                        \
        if (0)                                                                \
                no_printk("[%-5.5s%5u] " FMT "\n",                        \
                          current->comm, current->pid, ##__VA_ARGS__);        \
} while (0)
#endif

static struct kmem_cache *cred_jar;

/* init to 2 - one for init_task, one to ensure it is never freed */
static struct group_info init_groups = { .usage = REFCOUNT_INIT(2) };

/*
 * The initial credentials for the initial task
 */
struct cred init_cred = {
        .usage                        = ATOMIC_INIT(4),
        .uid                        = GLOBAL_ROOT_UID,
        .gid                        = GLOBAL_ROOT_GID,
        .suid                        = GLOBAL_ROOT_UID,
        .sgid                        = GLOBAL_ROOT_GID,
        .euid                        = GLOBAL_ROOT_UID,
        .egid                        = GLOBAL_ROOT_GID,
        .fsuid                        = GLOBAL_ROOT_UID,
        .fsgid                        = GLOBAL_ROOT_GID,
        .securebits                = SECUREBITS_DEFAULT,
        .cap_inheritable        = CAP_EMPTY_SET,
        .cap_permitted                = CAP_FULL_SET,
        .cap_effective                = CAP_FULL_SET,
        .cap_bset                = CAP_FULL_SET,
        .user                        = INIT_USER,
        .user_ns                = &init_user_ns,
        .group_info                = &init_groups,
        .ucounts                = &init_ucounts,
};

/*
 * The RCU callback to actually dispose of a set of credentials
 */
static void put_cred_rcu(struct rcu_head *rcu)
{
        struct cred *cred = container_of(rcu, struct cred, rcu);

        kdebug("put_cred_rcu(%p)", cred);

        if (atomic_long_read(&cred->usage) != 0)
                panic("CRED: put_cred_rcu() sees %p with usage %ld\n",
                      cred, atomic_long_read(&cred->usage));

        security_cred_free(cred);
        key_put(cred->session_keyring);
        key_put(cred->process_keyring);
        key_put(cred->thread_keyring);
        key_put(cred->request_key_auth);
        if (cred->group_info)
                put_group_info(cred->group_info);
        free_uid(cred->user);
        if (cred->ucounts)
                put_ucounts(cred->ucounts);
        put_user_ns(cred->user_ns);
        kmem_cache_free(cred_jar, cred);
}

/**
 * __put_cred - Destroy a set of credentials
 * @cred: The record to release
 *
 * Destroy a set of credentials on which no references remain.
 */
void __put_cred(struct cred *cred)
{
        kdebug("__put_cred(%p{%ld})", cred,
               atomic_long_read(&cred->usage));

        BUG_ON(atomic_long_read(&cred->usage) != 0);
        BUG_ON(cred == current->cred);
        BUG_ON(cred == current->real_cred);

        if (cred->non_rcu)
                put_cred_rcu(&cred->rcu);
        else
                call_rcu(&cred->rcu, put_cred_rcu);
}
EXPORT_SYMBOL(__put_cred);

/*
 * Clean up a task's credentials when it exits
 */
void exit_creds(struct task_struct *tsk)
{
        struct cred *real_cred, *cred;

        kdebug("exit_creds(%u,%p,%p,{%ld})", tsk->pid, tsk->real_cred, tsk->cred,
               atomic_long_read(&tsk->cred->usage));

        real_cred = (struct cred *) tsk->real_cred;
        tsk->real_cred = NULL;

        cred = (struct cred *) tsk->cred;
        tsk->cred = NULL;

        if (real_cred == cred) {
                put_cred_many(cred, 2);
        } else {
                put_cred(real_cred);
                put_cred(cred);
        }

#ifdef CONFIG_KEYS_REQUEST_CACHE
        key_put(tsk->cached_requested_key);
        tsk->cached_requested_key = NULL;
#endif
}

/**
 * get_task_cred - Get another task's objective credentials
 * @task: The task to query
 *
 * Get the objective credentials of a task, pinning them so that they can't go
 * away.  Accessing a task's credentials directly is not permitted.
 *
 * The caller must also make sure task doesn't get deleted, either by holding a
 * ref on task or by holding tasklist_lock to prevent it from being unlinked.
 */
const struct cred *get_task_cred(struct task_struct *task)
{
        const struct cred *cred;

        rcu_read_lock();

        do {
                cred = __task_cred((task));
                BUG_ON(!cred);
        } while (!get_cred_rcu(cred));

        rcu_read_unlock();
        return cred;
}
EXPORT_SYMBOL(get_task_cred);

/*
 * Allocate blank credentials, such that the credentials can be filled in at a
 * later date without risk of ENOMEM.
 */
struct cred *cred_alloc_blank(void)
{
        struct cred *new;

        new = kmem_cache_zalloc(cred_jar, GFP_KERNEL);
        if (!new)
                return NULL;

        atomic_long_set(&new->usage, 1);
        if (security_cred_alloc_blank(new, GFP_KERNEL_ACCOUNT) < 0)
                goto error;

        return new;

error:
        abort_creds(new);
        return NULL;
}

/**
 * prepare_creds - Prepare a new set of credentials for modification
 *
 * Prepare a new set of task credentials for modification.  A task's creds
 * shouldn't generally be modified directly, therefore this function is used to
 * prepare a new copy, which the caller then modifies and then commits by
 * calling commit_creds().
 *
 * Preparation involves making a copy of the objective creds for modification.
 *
 * Returns a pointer to the new creds-to-be if successful, NULL otherwise.
 *
 * Call commit_creds() or abort_creds() to clean up.
 */
struct cred *prepare_creds(void)
{
        struct task_struct *task = current;
        const struct cred *old;
        struct cred *new;

        new = kmem_cache_alloc(cred_jar, GFP_KERNEL);
        if (!new)
                return NULL;

        kdebug("prepare_creds() alloc %p", new);

        old = task->cred;
        memcpy(new, old, sizeof(struct cred));

        new->non_rcu = 0;
        atomic_long_set(&new->usage, 1);
        get_group_info(new->group_info);
        get_uid(new->user);
        get_user_ns(new->user_ns);

#ifdef CONFIG_KEYS
        key_get(new->session_keyring);
        key_get(new->process_keyring);
        key_get(new->thread_keyring);
        key_get(new->request_key_auth);
#endif

#ifdef CONFIG_SECURITY
        new->security = NULL;
#endif

        new->ucounts = get_ucounts(new->ucounts);
        if (!new->ucounts)
                goto error;

        if (security_prepare_creds(new, old, GFP_KERNEL_ACCOUNT) < 0)
                goto error;

        return new;

error:
        abort_creds(new);
        return NULL;
}
EXPORT_SYMBOL(prepare_creds);

/*
 * Prepare credentials for current to perform an execve()
 * - The caller must hold ->cred_guard_mutex
 */
struct cred *prepare_exec_creds(void)
{
        struct cred *new;

        new = prepare_creds();
        if (!new)
                return new;

#ifdef CONFIG_KEYS
        /* newly exec'd tasks don't get a thread keyring */
        key_put(new->thread_keyring);
        new->thread_keyring = NULL;

        /* inherit the session keyring; new process keyring */
        key_put(new->process_keyring);
        new->process_keyring = NULL;
#endif

        new->suid = new->fsuid = new->euid;
        new->sgid = new->fsgid = new->egid;

        return new;
}

/*
 * Copy credentials for the new process created by fork()
 *
 * We share if we can, but under some circumstances we have to generate a new
 * set.
 *
 * The new process gets the current process's subjective credentials as its
 * objective and subjective credentials
 */
int copy_creds(struct task_struct *p, unsigned long clone_flags)
{
        struct cred *new;
        int ret;

#ifdef CONFIG_KEYS_REQUEST_CACHE
        p->cached_requested_key = NULL;
#endif

        if (
#ifdef CONFIG_KEYS
                !p->cred->thread_keyring &&
#endif
                clone_flags & CLONE_THREAD
            ) {
                p->real_cred = get_cred_many(p->cred, 2);
                kdebug("share_creds(%p{%ld})",
                       p->cred, atomic_long_read(&p->cred->usage));
                inc_rlimit_ucounts(task_ucounts(p), UCOUNT_RLIMIT_NPROC, 1);
                return 0;
        }

        new = prepare_creds();
        if (!new)
                return -ENOMEM;

        if (clone_flags & CLONE_NEWUSER) {
                ret = create_user_ns(new);
                if (ret < 0)
                        goto error_put;
                ret = set_cred_ucounts(new);
                if (ret < 0)
                        goto error_put;
        }

#ifdef CONFIG_KEYS
        /* new threads get their own thread keyrings if their parent already
         * had one */
        if (new->thread_keyring) {
                key_put(new->thread_keyring);
                new->thread_keyring = NULL;
                if (clone_flags & CLONE_THREAD)
                        install_thread_keyring_to_cred(new);
        }

        /* The process keyring is only shared between the threads in a process;
         * anything outside of those threads doesn't inherit.
         */
        if (!(clone_flags & CLONE_THREAD)) {
                key_put(new->process_keyring);
                new->process_keyring = NULL;
        }
#endif

        p->cred = p->real_cred = get_cred(new);
        inc_rlimit_ucounts(task_ucounts(p), UCOUNT_RLIMIT_NPROC, 1);
        return 0;

error_put:
        put_cred(new);
        return ret;
}

static bool cred_cap_issubset(const struct cred *set, const struct cred *subset)
{
        const struct user_namespace *set_ns = set->user_ns;
        const struct user_namespace *subset_ns = subset->user_ns;

        /* If the two credentials are in the same user namespace see if
         * the capabilities of subset are a subset of set.
         */
        if (set_ns == subset_ns)
                return cap_issubset(subset->cap_permitted, set->cap_permitted);

        /* The credentials are in a different user namespaces
         * therefore one is a subset of the other only if a set is an
         * ancestor of subset and set->euid is owner of subset or one
         * of subsets ancestors.
         */
        for (;subset_ns != &init_user_ns; subset_ns = subset_ns->parent) {
                if ((set_ns == subset_ns->parent)  &&
                    uid_eq(subset_ns->owner, set->euid))
                        return true;
        }

        return false;
}

/**
 * commit_creds - Install new credentials upon the current task
 * @new: The credentials to be assigned
 *
 * Install a new set of credentials to the current task, using RCU to replace
 * the old set.  Both the objective and the subjective credentials pointers are
 * updated.  This function may not be called if the subjective credentials are
 * in an overridden state.
 *
 * This function eats the caller's reference to the new credentials.
 *
 * Always returns 0 thus allowing this function to be tail-called at the end
 * of, say, sys_setgid().
 */
int commit_creds(struct cred *new)
{
        struct task_struct *task = current;
        const struct cred *old = task->real_cred;

        kdebug("commit_creds(%p{%ld})", new,
               atomic_long_read(&new->usage));

        BUG_ON(task->cred != old);
        BUG_ON(atomic_long_read(&new->usage) < 1);

        get_cred(new); /* we will require a ref for the subj creds too */

        /* dumpability changes */
        if (!uid_eq(old->euid, new->euid) ||
            !gid_eq(old->egid, new->egid) ||
            !uid_eq(old->fsuid, new->fsuid) ||
            !gid_eq(old->fsgid, new->fsgid) ||
            !cred_cap_issubset(old, new)) {
                if (task->mm)
                        set_dumpable(task->mm, suid_dumpable);
                task->pdeath_signal = 0;
                /*
                 * If a task drops privileges and becomes nondumpable,
                 * the dumpability change must become visible before
                 * the credential change; otherwise, a __ptrace_may_access()
                 * racing with this change may be able to attach to a task it
                 * shouldn't be able to attach to (as if the task had dropped
                 * privileges without becoming nondumpable).
                 * Pairs with a read barrier in __ptrace_may_access().
                 */
                smp_wmb();
        }

        /* alter the thread keyring */
        if (!uid_eq(new->fsuid, old->fsuid))
                key_fsuid_changed(new);
        if (!gid_eq(new->fsgid, old->fsgid))
                key_fsgid_changed(new);

        /* do it
         * RLIMIT_NPROC limits on user->processes have already been checked
         * in set_user().
         */
        if (new->user != old->user || new->user_ns != old->user_ns)
                inc_rlimit_ucounts(new->ucounts, UCOUNT_RLIMIT_NPROC, 1);
        rcu_assign_pointer(task->real_cred, new);
        rcu_assign_pointer(task->cred, new);
        if (new->user != old->user || new->user_ns != old->user_ns)
                dec_rlimit_ucounts(old->ucounts, UCOUNT_RLIMIT_NPROC, 1);

        /* send notifications */
        if (!uid_eq(new->uid,   old->uid)  ||
            !uid_eq(new->euid,  old->euid) ||
            !uid_eq(new->suid,  old->suid) ||
            !uid_eq(new->fsuid, old->fsuid))
                proc_id_connector(task, PROC_EVENT_UID);

        if (!gid_eq(new->gid,   old->gid)  ||
            !gid_eq(new->egid,  old->egid) ||
            !gid_eq(new->sgid,  old->sgid) ||
            !gid_eq(new->fsgid, old->fsgid))
                proc_id_connector(task, PROC_EVENT_GID);

        /* release the old obj and subj refs both */
        put_cred_many(old, 2);
        return 0;
}
EXPORT_SYMBOL(commit_creds);

/**
 * abort_creds - Discard a set of credentials and unlock the current task
 * @new: The credentials that were going to be applied
 *
 * Discard a set of credentials that were under construction and unlock the
 * current task.
 */
void abort_creds(struct cred *new)
{
        kdebug("abort_creds(%p{%ld})", new,
               atomic_long_read(&new->usage));

        BUG_ON(atomic_long_read(&new->usage) < 1);
        put_cred(new);
}
EXPORT_SYMBOL(abort_creds);

/**
 * cred_fscmp - Compare two credentials with respect to filesystem access.
 * @a: The first credential
 * @b: The second credential
 *
 * cred_cmp() will return zero if both credentials have the same
 * fsuid, fsgid, and supplementary groups.  That is, if they will both
 * provide the same access to files based on mode/uid/gid.
 * If the credentials are different, then either -1 or 1 will
 * be returned depending on whether @a comes before or after @b
 * respectively in an arbitrary, but stable, ordering of credentials.
 *
 * Return: -1, 0, or 1 depending on comparison
 */
int cred_fscmp(const struct cred *a, const struct cred *b)
{
        struct group_info *ga, *gb;
        int g;

        if (a == b)
                return 0;
        if (uid_lt(a->fsuid, b->fsuid))
                return -1;
        if (uid_gt(a->fsuid, b->fsuid))
                return 1;

        if (gid_lt(a->fsgid, b->fsgid))
                return -1;
        if (gid_gt(a->fsgid, b->fsgid))
                return 1;

        ga = a->group_info;
        gb = b->group_info;
        if (ga == gb)
                return 0;
        if (ga == NULL)
                return -1;
        if (gb == NULL)
                return 1;
        if (ga->ngroups < gb->ngroups)
                return -1;
        if (ga->ngroups > gb->ngroups)
                return 1;

        for (g = 0; g < ga->ngroups; g++) {
                if (gid_lt(ga->gid[g], gb->gid[g]))
                        return -1;
                if (gid_gt(ga->gid[g], gb->gid[g]))
                        return 1;
        }
        return 0;
}
EXPORT_SYMBOL(cred_fscmp);

int set_cred_ucounts(struct cred *new)
{
        struct ucounts *new_ucounts, *old_ucounts = new->ucounts;

        /*
         * This optimization is needed because alloc_ucounts() uses locks
         * for table lookups.
         */
        if (old_ucounts->ns == new->user_ns && uid_eq(old_ucounts->uid, new->uid))
                return 0;

        if (!(new_ucounts = alloc_ucounts(new->user_ns, new->uid)))
                return -EAGAIN;

        new->ucounts = new_ucounts;
        put_ucounts(old_ucounts);

        return 0;
}

/*
 * initialise the credentials stuff
 */
void __init cred_init(void)
{
        /* allocate a slab in which we can store credentials */
        cred_jar = KMEM_CACHE(cred,
                              SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT);
}

/**
 * prepare_kernel_cred - Prepare a set of credentials for a kernel service
 * @daemon: A userspace daemon to be used as a reference
 *
 * Prepare a set of credentials for a kernel service.  This can then be used to
 * override a task's own credentials so that work can be done on behalf of that
 * task that requires a different subjective context.
 *
 * @daemon is used to provide a base cred, with the security data derived from
 * that; if this is "&init_task", they'll be set to 0, no groups, full
 * capabilities, and no keys.
 *
 * The caller may change these controls afterwards if desired.
 *
 * Returns the new credentials or NULL if out of memory.
 */
struct cred *prepare_kernel_cred(struct task_struct *daemon)
{
        const struct cred *old;
        struct cred *new;

        if (WARN_ON_ONCE(!daemon))
                return NULL;

        new = kmem_cache_alloc(cred_jar, GFP_KERNEL);
        if (!new)
                return NULL;

        kdebug("prepare_kernel_cred() alloc %p", new);

        old = get_task_cred(daemon);

        *new = *old;
        new->non_rcu = 0;
        atomic_long_set(&new->usage, 1);
        get_uid(new->user);
        get_user_ns(new->user_ns);
        get_group_info(new->group_info);

#ifdef CONFIG_KEYS
        new->session_keyring = NULL;
        new->process_keyring = NULL;
        new->thread_keyring = NULL;
        new->request_key_auth = NULL;
        new->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING;
#endif

#ifdef CONFIG_SECURITY
        new->security = NULL;
#endif
        new->ucounts = get_ucounts(new->ucounts);
        if (!new->ucounts)
                goto error;

        if (security_prepare_creds(new, old, GFP_KERNEL_ACCOUNT) < 0)
                goto error;

        put_cred(old);
        return new;

error:
        put_cred(new);
        put_cred(old);
        return NULL;
}
EXPORT_SYMBOL(prepare_kernel_cred);

/**
 * set_security_override - Set the security ID in a set of credentials
 * @new: The credentials to alter
 * @secid: The LSM security ID to set
 *
 * Set the LSM security ID in a set of credentials so that the subjective
 * security is overridden when an alternative set of credentials is used.
 */
int set_security_override(struct cred *new, u32 secid)
{
        return security_kernel_act_as(new, secid);
}
EXPORT_SYMBOL(set_security_override);

/**
 * set_security_override_from_ctx - Set the security ID in a set of credentials
 * @new: The credentials to alter
 * @secctx: The LSM security context to generate the security ID from.
 *
 * Set the LSM security ID in a set of credentials so that the subjective
 * security is overridden when an alternative set of credentials is used.  The
 * security ID is specified in string form as a security context to be
 * interpreted by the LSM.
 */
int set_security_override_from_ctx(struct cred *new, const char *secctx)
{
        u32 secid;
        int ret;

        ret = security_secctx_to_secid(secctx, strlen(secctx), &secid);
        if (ret < 0)
                return ret;

        return set_security_override(new, secid);
}
EXPORT_SYMBOL(set_security_override_from_ctx);

/**
 * set_create_files_as - Set the LSM file create context in a set of credentials
 * @new: The credentials to alter
 * @inode: The inode to take the context from
 *
 * Change the LSM file creation context in a set of credentials to be the same
 * as the object context of the specified inode, so that the new inodes have
 * the same MAC context as that inode.
 */
int set_create_files_as(struct cred *new, struct inode *inode)
{
        if (!uid_valid(inode->i_uid) || !gid_valid(inode->i_gid))
                return -EINVAL;
        new->fsuid = inode->i_uid;
        new->fsgid = inode->i_gid;
        return security_kernel_create_files_as(new, inode);
}
EXPORT_SYMBOL(set_create_files_as);










































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   35 
   35 





   24 
   24 
































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * drivers/net/bond/bond_options.c - bonding options
 * Copyright (c) 2013 Jiri Pirko <jiri@resnulli.us>
 * Copyright (c) 2013 Scott Feldman <sfeldma@cumulusnetworks.com>
 */

#include <linux/errno.h>
#include <linux/if.h>
#include <linux/netdevice.h>
#include <linux/spinlock.h>
#include <linux/rcupdate.h>
#include <linux/ctype.h>
#include <linux/inet.h>
#include <linux/sched/signal.h>

#include <net/bonding.h>
#include <net/ndisc.h>

static int bond_option_active_slave_set(struct bonding *bond,
                                        const struct bond_opt_value *newval);
static int bond_option_miimon_set(struct bonding *bond,
                                  const struct bond_opt_value *newval);
static int bond_option_updelay_set(struct bonding *bond,
                                   const struct bond_opt_value *newval);
static int bond_option_downdelay_set(struct bonding *bond,
                                     const struct bond_opt_value *newval);
static int bond_option_peer_notif_delay_set(struct bonding *bond,
                                            const struct bond_opt_value *newval);
static int bond_option_use_carrier_set(struct bonding *bond,
                                       const struct bond_opt_value *newval);
static int bond_option_arp_interval_set(struct bonding *bond,
                                        const struct bond_opt_value *newval);
static int bond_option_arp_ip_target_add(struct bonding *bond, __be32 target);
static int bond_option_arp_ip_target_rem(struct bonding *bond, __be32 target);
static int bond_option_arp_ip_targets_set(struct bonding *bond,
                                          const struct bond_opt_value *newval);
static int bond_option_ns_ip6_targets_set(struct bonding *bond,
                                          const struct bond_opt_value *newval);
static int bond_option_arp_validate_set(struct bonding *bond,
                                        const struct bond_opt_value *newval);
static int bond_option_arp_all_targets_set(struct bonding *bond,
                                           const struct bond_opt_value *newval);
static int bond_option_prio_set(struct bonding *bond,
                                const struct bond_opt_value *newval);
static int bond_option_primary_set(struct bonding *bond,
                                   const struct bond_opt_value *newval);
static int bond_option_primary_reselect_set(struct bonding *bond,
                                            const struct bond_opt_value *newval);
static int bond_option_fail_over_mac_set(struct bonding *bond,
                                         const struct bond_opt_value *newval);
static int bond_option_xmit_hash_policy_set(struct bonding *bond,
                                            const struct bond_opt_value *newval);
static int bond_option_resend_igmp_set(struct bonding *bond,
                                       const struct bond_opt_value *newval);
static int bond_option_num_peer_notif_set(struct bonding *bond,
                                          const struct bond_opt_value *newval);
static int bond_option_all_slaves_active_set(struct bonding *bond,
                                             const struct bond_opt_value *newval);
static int bond_option_min_links_set(struct bonding *bond,
                                     const struct bond_opt_value *newval);
static int bond_option_lp_interval_set(struct bonding *bond,
                                       const struct bond_opt_value *newval);
static int bond_option_pps_set(struct bonding *bond,
                               const struct bond_opt_value *newval);
static int bond_option_lacp_active_set(struct bonding *bond,
                                       const struct bond_opt_value *newval);
static int bond_option_lacp_rate_set(struct bonding *bond,
                                     const struct bond_opt_value *newval);
static int bond_option_ad_select_set(struct bonding *bond,
                                     const struct bond_opt_value *newval);
static int bond_option_queue_id_set(struct bonding *bond,
                                    const struct bond_opt_value *newval);
static int bond_option_mode_set(struct bonding *bond,
                                const struct bond_opt_value *newval);
static int bond_option_slaves_set(struct bonding *bond,
                                  const struct bond_opt_value *newval);
static int bond_option_tlb_dynamic_lb_set(struct bonding *bond,
                                  const struct bond_opt_value *newval);
static int bond_option_ad_actor_sys_prio_set(struct bonding *bond,
                                             const struct bond_opt_value *newval);
static int bond_option_ad_actor_system_set(struct bonding *bond,
                                           const struct bond_opt_value *newval);
static int bond_option_ad_user_port_key_set(struct bonding *bond,
                                            const struct bond_opt_value *newval);
static int bond_option_missed_max_set(struct bonding *bond,
                                      const struct bond_opt_value *newval);
static int bond_option_coupled_control_set(struct bonding *bond,
                                           const struct bond_opt_value *newval);

static const struct bond_opt_value bond_mode_tbl[] = {
        { "balance-rr",    BOND_MODE_ROUNDROBIN,   BOND_VALFLAG_DEFAULT},
        { "active-backup", BOND_MODE_ACTIVEBACKUP, 0},
        { "balance-xor",   BOND_MODE_XOR,          0},
        { "broadcast",     BOND_MODE_BROADCAST,    0},
        { "802.3ad",       BOND_MODE_8023AD,       0},
        { "balance-tlb",   BOND_MODE_TLB,          0},
        { "balance-alb",   BOND_MODE_ALB,          0},
        { NULL,            -1,                     0},
};

static const struct bond_opt_value bond_pps_tbl[] = {
        { "default", 1,         BOND_VALFLAG_DEFAULT},
        { "maxval",  USHRT_MAX, BOND_VALFLAG_MAX},
        { NULL,      -1,        0},
};

static const struct bond_opt_value bond_xmit_hashtype_tbl[] = {
        { "layer2",      BOND_XMIT_POLICY_LAYER2,      BOND_VALFLAG_DEFAULT},
        { "layer3+4",    BOND_XMIT_POLICY_LAYER34,     0},
        { "layer2+3",    BOND_XMIT_POLICY_LAYER23,     0},
        { "encap2+3",    BOND_XMIT_POLICY_ENCAP23,     0},
        { "encap3+4",    BOND_XMIT_POLICY_ENCAP34,     0},
        { "vlan+srcmac", BOND_XMIT_POLICY_VLAN_SRCMAC, 0},
        { NULL,          -1,                           0},
};

static const struct bond_opt_value bond_arp_validate_tbl[] = {
        { "none",                BOND_ARP_VALIDATE_NONE,                BOND_VALFLAG_DEFAULT},
        { "active",                BOND_ARP_VALIDATE_ACTIVE,        0},
        { "backup",                BOND_ARP_VALIDATE_BACKUP,        0},
        { "all",                BOND_ARP_VALIDATE_ALL,                0},
        { "filter",                BOND_ARP_FILTER,                0},
        { "filter_active",        BOND_ARP_FILTER_ACTIVE,                0},
        { "filter_backup",        BOND_ARP_FILTER_BACKUP,                0},
        { NULL,                        -1,                                0},
};

static const struct bond_opt_value bond_arp_all_targets_tbl[] = {
        { "any", BOND_ARP_TARGETS_ANY, BOND_VALFLAG_DEFAULT},
        { "all", BOND_ARP_TARGETS_ALL, 0},
        { NULL,  -1,                   0},
};

static const struct bond_opt_value bond_fail_over_mac_tbl[] = {
        { "none",   BOND_FOM_NONE,   BOND_VALFLAG_DEFAULT},
        { "active", BOND_FOM_ACTIVE, 0},
        { "follow", BOND_FOM_FOLLOW, 0},
        { NULL,     -1,              0},
};

static const struct bond_opt_value bond_intmax_tbl[] = {
        { "off",     0,       BOND_VALFLAG_DEFAULT},
        { "maxval",  INT_MAX, BOND_VALFLAG_MAX},
        { NULL,      -1,      0}
};

static const struct bond_opt_value bond_lacp_active[] = {
        { "off", 0,  0},
        { "on",  1,  BOND_VALFLAG_DEFAULT},
        { NULL,  -1, 0}
};

static const struct bond_opt_value bond_lacp_rate_tbl[] = {
        { "slow", AD_LACP_SLOW, 0},
        { "fast", AD_LACP_FAST, 0},
        { NULL,   -1,           0},
};

static const struct bond_opt_value bond_ad_select_tbl[] = {
        { "stable",    BOND_AD_STABLE,    BOND_VALFLAG_DEFAULT},
        { "bandwidth", BOND_AD_BANDWIDTH, 0},
        { "count",     BOND_AD_COUNT,     0},
        { NULL,        -1,                0},
};

static const struct bond_opt_value bond_num_peer_notif_tbl[] = {
        { "off",     0,   0},
        { "maxval",  255, BOND_VALFLAG_MAX},
        { "default", 1,   BOND_VALFLAG_DEFAULT},
        { NULL,      -1,  0}
};

static const struct bond_opt_value bond_peer_notif_delay_tbl[] = {
        { "off",     0,   0},
        { "maxval",  300000, BOND_VALFLAG_MAX},
        { NULL,      -1,  0}
};

static const struct bond_opt_value bond_primary_reselect_tbl[] = {
        { "always",  BOND_PRI_RESELECT_ALWAYS,  BOND_VALFLAG_DEFAULT},
        { "better",  BOND_PRI_RESELECT_BETTER,  0},
        { "failure", BOND_PRI_RESELECT_FAILURE, 0},
        { NULL,      -1},
};

static const struct bond_opt_value bond_use_carrier_tbl[] = {
        { "off", 0,  0},
        { "on",  1,  BOND_VALFLAG_DEFAULT},
        { NULL,  -1, 0}
};

static const struct bond_opt_value bond_all_slaves_active_tbl[] = {
        { "off", 0,  BOND_VALFLAG_DEFAULT},
        { "on",  1,  0},
        { NULL,  -1, 0}
};

static const struct bond_opt_value bond_resend_igmp_tbl[] = {
        { "off",     0,   0},
        { "maxval",  255, BOND_VALFLAG_MAX},
        { "default", 1,   BOND_VALFLAG_DEFAULT},
        { NULL,      -1,  0}
};

static const struct bond_opt_value bond_lp_interval_tbl[] = {
        { "minval",  1,       BOND_VALFLAG_MIN | BOND_VALFLAG_DEFAULT},
        { "maxval",  INT_MAX, BOND_VALFLAG_MAX},
        { NULL,      -1,      0},
};

static const struct bond_opt_value bond_tlb_dynamic_lb_tbl[] = {
        { "off", 0,  0},
        { "on",  1,  BOND_VALFLAG_DEFAULT},
        { NULL,  -1, 0}
};

static const struct bond_opt_value bond_ad_actor_sys_prio_tbl[] = {
        { "minval",  1,     BOND_VALFLAG_MIN},
        { "maxval",  65535, BOND_VALFLAG_MAX | BOND_VALFLAG_DEFAULT},
        { NULL,      -1,    0},
};

static const struct bond_opt_value bond_ad_user_port_key_tbl[] = {
        { "minval",  0,     BOND_VALFLAG_MIN | BOND_VALFLAG_DEFAULT},
        { "maxval",  1023,  BOND_VALFLAG_MAX},
        { NULL,      -1,    0},
};

static const struct bond_opt_value bond_missed_max_tbl[] = {
        { "minval",        1,        BOND_VALFLAG_MIN},
        { "maxval",        255,        BOND_VALFLAG_MAX},
        { "default",        2,        BOND_VALFLAG_DEFAULT},
        { NULL,                -1,        0},
};

static const struct bond_opt_value bond_coupled_control_tbl[] = {
        { "on",  1,  BOND_VALFLAG_DEFAULT},
        { "off", 0,  0},
        { NULL,  -1, 0},
};

static const struct bond_option bond_opts[BOND_OPT_LAST] = {
        [BOND_OPT_MODE] = {
                .id = BOND_OPT_MODE,
                .name = "mode",
                .desc = "bond device mode",
                .flags = BOND_OPTFLAG_NOSLAVES | BOND_OPTFLAG_IFDOWN,
                .values = bond_mode_tbl,
                .set = bond_option_mode_set
        },
        [BOND_OPT_PACKETS_PER_SLAVE] = {
                .id = BOND_OPT_PACKETS_PER_SLAVE,
                .name = "packets_per_slave",
                .desc = "Packets to send per slave in RR mode",
                .unsuppmodes = BOND_MODE_ALL_EX(BIT(BOND_MODE_ROUNDROBIN)),
                .values = bond_pps_tbl,
                .set = bond_option_pps_set
        },
        [BOND_OPT_XMIT_HASH] = {
                .id = BOND_OPT_XMIT_HASH,
                .name = "xmit_hash_policy",
                .desc = "balance-xor, 802.3ad, and tlb hashing method",
                .values = bond_xmit_hashtype_tbl,
                .set = bond_option_xmit_hash_policy_set
        },
        [BOND_OPT_ARP_VALIDATE] = {
                .id = BOND_OPT_ARP_VALIDATE,
                .name = "arp_validate",
                .desc = "validate src/dst of ARP probes",
                .unsuppmodes = BIT(BOND_MODE_8023AD) | BIT(BOND_MODE_TLB) |
                               BIT(BOND_MODE_ALB),
                .values = bond_arp_validate_tbl,
                .set = bond_option_arp_validate_set
        },
        [BOND_OPT_ARP_ALL_TARGETS] = {
                .id = BOND_OPT_ARP_ALL_TARGETS,
                .name = "arp_all_targets",
                .desc = "fail on any/all arp targets timeout",
                .values = bond_arp_all_targets_tbl,
                .set = bond_option_arp_all_targets_set
        },
        [BOND_OPT_FAIL_OVER_MAC] = {
                .id = BOND_OPT_FAIL_OVER_MAC,
                .name = "fail_over_mac",
                .desc = "For active-backup, do not set all slaves to the same MAC",
                .flags = BOND_OPTFLAG_NOSLAVES,
                .values = bond_fail_over_mac_tbl,
                .set = bond_option_fail_over_mac_set
        },
        [BOND_OPT_ARP_INTERVAL] = {
                .id = BOND_OPT_ARP_INTERVAL,
                .name = "arp_interval",
                .desc = "arp interval in milliseconds",
                .unsuppmodes = BIT(BOND_MODE_8023AD) | BIT(BOND_MODE_TLB) |
                               BIT(BOND_MODE_ALB),
                .values = bond_intmax_tbl,
                .set = bond_option_arp_interval_set
        },
        [BOND_OPT_MISSED_MAX] = {
                .id = BOND_OPT_MISSED_MAX,
                .name = "arp_missed_max",
                .desc = "Maximum number of missed ARP interval",
                .unsuppmodes = BIT(BOND_MODE_8023AD) | BIT(BOND_MODE_TLB) |
                               BIT(BOND_MODE_ALB),
                .values = bond_missed_max_tbl,
                .set = bond_option_missed_max_set
        },
        [BOND_OPT_ARP_TARGETS] = {
                .id = BOND_OPT_ARP_TARGETS,
                .name = "arp_ip_target",
                .desc = "arp targets in n.n.n.n form",
                .flags = BOND_OPTFLAG_RAWVAL,
                .set = bond_option_arp_ip_targets_set
        },
        [BOND_OPT_NS_TARGETS] = {
                .id = BOND_OPT_NS_TARGETS,
                .name = "ns_ip6_target",
                .desc = "NS targets in ffff:ffff::ffff:ffff form",
                .flags = BOND_OPTFLAG_RAWVAL,
                .set = bond_option_ns_ip6_targets_set
        },
        [BOND_OPT_DOWNDELAY] = {
                .id = BOND_OPT_DOWNDELAY,
                .name = "downdelay",
                .desc = "Delay before considering link down, in milliseconds",
                .values = bond_intmax_tbl,
                .set = bond_option_downdelay_set
        },
        [BOND_OPT_UPDELAY] = {
                .id = BOND_OPT_UPDELAY,
                .name = "updelay",
                .desc = "Delay before considering link up, in milliseconds",
                .values = bond_intmax_tbl,
                .set = bond_option_updelay_set
        },
        [BOND_OPT_LACP_ACTIVE] = {
                .id = BOND_OPT_LACP_ACTIVE,
                .name = "lacp_active",
                .desc = "Send LACPDU frames with configured lacp rate or acts as speak when spoken to",
                .flags = BOND_OPTFLAG_IFDOWN,
                .unsuppmodes = BOND_MODE_ALL_EX(BIT(BOND_MODE_8023AD)),
                .values = bond_lacp_active,
                .set = bond_option_lacp_active_set
        },
        [BOND_OPT_LACP_RATE] = {
                .id = BOND_OPT_LACP_RATE,
                .name = "lacp_rate",
                .desc = "LACPDU tx rate to request from 802.3ad partner",
                .flags = BOND_OPTFLAG_IFDOWN,
                .unsuppmodes = BOND_MODE_ALL_EX(BIT(BOND_MODE_8023AD)),
                .values = bond_lacp_rate_tbl,
                .set = bond_option_lacp_rate_set
        },
        [BOND_OPT_MINLINKS] = {
                .id = BOND_OPT_MINLINKS,
                .name = "min_links",
                .desc = "Minimum number of available links before turning on carrier",
                .values = bond_intmax_tbl,
                .set = bond_option_min_links_set
        },
        [BOND_OPT_AD_SELECT] = {
                .id = BOND_OPT_AD_SELECT,
                .name = "ad_select",
                .desc = "803.ad aggregation selection logic",
                .flags = BOND_OPTFLAG_IFDOWN,
                .values = bond_ad_select_tbl,
                .set = bond_option_ad_select_set
        },
        [BOND_OPT_NUM_PEER_NOTIF] = {
                .id = BOND_OPT_NUM_PEER_NOTIF,
                .name = "num_unsol_na",
                .desc = "Number of peer notifications to send on failover event",
                .values = bond_num_peer_notif_tbl,
                .set = bond_option_num_peer_notif_set
        },
        [BOND_OPT_MIIMON] = {
                .id = BOND_OPT_MIIMON,
                .name = "miimon",
                .desc = "Link check interval in milliseconds",
                .values = bond_intmax_tbl,
                .set = bond_option_miimon_set
        },
        [BOND_OPT_PRIO] = {
                .id = BOND_OPT_PRIO,
                .name = "prio",
                .desc = "Link priority for failover re-selection",
                .flags = BOND_OPTFLAG_RAWVAL,
                .unsuppmodes = BOND_MODE_ALL_EX(BIT(BOND_MODE_ACTIVEBACKUP) |
                                                BIT(BOND_MODE_TLB) |
                                                BIT(BOND_MODE_ALB)),
                .set = bond_option_prio_set
        },
        [BOND_OPT_PRIMARY] = {
                .id = BOND_OPT_PRIMARY,
                .name = "primary",
                .desc = "Primary network device to use",
                .flags = BOND_OPTFLAG_RAWVAL,
                .unsuppmodes = BOND_MODE_ALL_EX(BIT(BOND_MODE_ACTIVEBACKUP) |
                                                BIT(BOND_MODE_TLB) |
                                                BIT(BOND_MODE_ALB)),
                .set = bond_option_primary_set
        },
        [BOND_OPT_PRIMARY_RESELECT] = {
                .id = BOND_OPT_PRIMARY_RESELECT,
                .name = "primary_reselect",
                .desc = "Reselect primary slave once it comes up",
                .values = bond_primary_reselect_tbl,
                .set = bond_option_primary_reselect_set
        },
        [BOND_OPT_USE_CARRIER] = {
                .id = BOND_OPT_USE_CARRIER,
                .name = "use_carrier",
                .desc = "Use netif_carrier_ok (vs MII ioctls) in miimon",
                .values = bond_use_carrier_tbl,
                .set = bond_option_use_carrier_set
        },
        [BOND_OPT_ACTIVE_SLAVE] = {
                .id = BOND_OPT_ACTIVE_SLAVE,
                .name = "active_slave",
                .desc = "Currently active slave",
                .flags = BOND_OPTFLAG_RAWVAL,
                .unsuppmodes = BOND_MODE_ALL_EX(BIT(BOND_MODE_ACTIVEBACKUP) |
                                                BIT(BOND_MODE_TLB) |
                                                BIT(BOND_MODE_ALB)),
                .set = bond_option_active_slave_set
        },
        [BOND_OPT_QUEUE_ID] = {
                .id = BOND_OPT_QUEUE_ID,
                .name = "queue_id",
                .desc = "Set queue id of a slave",
                .flags = BOND_OPTFLAG_RAWVAL,
                .set = bond_option_queue_id_set
        },
        [BOND_OPT_ALL_SLAVES_ACTIVE] = {
                .id = BOND_OPT_ALL_SLAVES_ACTIVE,
                .name = "all_slaves_active",
                .desc = "Keep all frames received on an interface by setting active flag for all slaves",
                .values = bond_all_slaves_active_tbl,
                .set = bond_option_all_slaves_active_set
        },
        [BOND_OPT_RESEND_IGMP] = {
                .id = BOND_OPT_RESEND_IGMP,
                .name = "resend_igmp",
                .desc = "Number of IGMP membership reports to send on link failure",
                .values = bond_resend_igmp_tbl,
                .set = bond_option_resend_igmp_set
        },
        [BOND_OPT_LP_INTERVAL] = {
                .id = BOND_OPT_LP_INTERVAL,
                .name = "lp_interval",
                .desc = "The number of seconds between instances where the bonding driver sends learning packets to each slave's peer switch",
                .values = bond_lp_interval_tbl,
                .set = bond_option_lp_interval_set
        },
        [BOND_OPT_SLAVES] = {
                .id = BOND_OPT_SLAVES,
                .name = "slaves",
                .desc = "Slave membership management",
                .flags = BOND_OPTFLAG_RAWVAL,
                .set = bond_option_slaves_set
        },
        [BOND_OPT_TLB_DYNAMIC_LB] = {
                .id = BOND_OPT_TLB_DYNAMIC_LB,
                .name = "tlb_dynamic_lb",
                .desc = "Enable dynamic flow shuffling",
                .unsuppmodes = BOND_MODE_ALL_EX(BIT(BOND_MODE_TLB) | BIT(BOND_MODE_ALB)),
                .values = bond_tlb_dynamic_lb_tbl,
                .flags = BOND_OPTFLAG_IFDOWN,
                .set = bond_option_tlb_dynamic_lb_set,
        },
        [BOND_OPT_AD_ACTOR_SYS_PRIO] = {
                .id = BOND_OPT_AD_ACTOR_SYS_PRIO,
                .name = "ad_actor_sys_prio",
                .unsuppmodes = BOND_MODE_ALL_EX(BIT(BOND_MODE_8023AD)),
                .values = bond_ad_actor_sys_prio_tbl,
                .set = bond_option_ad_actor_sys_prio_set,
        },
        [BOND_OPT_AD_ACTOR_SYSTEM] = {
                .id = BOND_OPT_AD_ACTOR_SYSTEM,
                .name = "ad_actor_system",
                .unsuppmodes = BOND_MODE_ALL_EX(BIT(BOND_MODE_8023AD)),
                .flags = BOND_OPTFLAG_RAWVAL,
                .set = bond_option_ad_actor_system_set,
        },
        [BOND_OPT_AD_USER_PORT_KEY] = {
                .id = BOND_OPT_AD_USER_PORT_KEY,
                .name = "ad_user_port_key",
                .unsuppmodes = BOND_MODE_ALL_EX(BIT(BOND_MODE_8023AD)),
                .flags = BOND_OPTFLAG_IFDOWN,
                .values = bond_ad_user_port_key_tbl,
                .set = bond_option_ad_user_port_key_set,
        },
        [BOND_OPT_NUM_PEER_NOTIF_ALIAS] = {
                .id = BOND_OPT_NUM_PEER_NOTIF_ALIAS,
                .name = "num_grat_arp",
                .desc = "Number of peer notifications to send on failover event",
                .values = bond_num_peer_notif_tbl,
                .set = bond_option_num_peer_notif_set
        },
        [BOND_OPT_PEER_NOTIF_DELAY] = {
                .id = BOND_OPT_PEER_NOTIF_DELAY,
                .name = "peer_notif_delay",
                .desc = "Delay between each peer notification on failover event, in milliseconds",
                .values = bond_peer_notif_delay_tbl,
                .set = bond_option_peer_notif_delay_set
        },
        [BOND_OPT_COUPLED_CONTROL] = {
                .id = BOND_OPT_COUPLED_CONTROL,
                .name = "coupled_control",
                .desc = "Opt into using coupled control MUX for LACP states",
                .unsuppmodes = BOND_MODE_ALL_EX(BIT(BOND_MODE_8023AD)),
                .flags = BOND_OPTFLAG_IFDOWN,
                .values = bond_coupled_control_tbl,
                .set = bond_option_coupled_control_set,
        }
};

/* Searches for an option by name */
const struct bond_option *bond_opt_get_by_name(const char *name)
{
        const struct bond_option *opt;
        int option;

        for (option = 0; option < BOND_OPT_LAST; option++) {
                opt = bond_opt_get(option);
                if (opt && !strcmp(opt->name, name))
                        return opt;
        }

        return NULL;
}

/* Searches for a value in opt's values[] table */
const struct bond_opt_value *bond_opt_get_val(unsigned int option, u64 val)
{
        const struct bond_option *opt;
        int i;

        opt = bond_opt_get(option);
        if (WARN_ON(!opt))
                return NULL;
        for (i = 0; opt->values && opt->values[i].string; i++)
                if (opt->values[i].value == val)
                        return &opt->values[i];

        return NULL;
}

/* Searches for a value in opt's values[] table which matches the flagmask */
static const struct bond_opt_value *bond_opt_get_flags(const struct bond_option *opt,
                                                       u32 flagmask)
{
        int i;

        for (i = 0; opt->values && opt->values[i].string; i++)
                if (opt->values[i].flags & flagmask)
                        return &opt->values[i];

        return NULL;
}

/* If maxval is missing then there's no range to check. In case minval is
 * missing then it's considered to be 0.
 */
static bool bond_opt_check_range(const struct bond_option *opt, u64 val)
{
        const struct bond_opt_value *minval, *maxval;

        minval = bond_opt_get_flags(opt, BOND_VALFLAG_MIN);
        maxval = bond_opt_get_flags(opt, BOND_VALFLAG_MAX);
        if (!maxval || (minval && val < minval->value) || val > maxval->value)
                return false;

        return true;
}

/**
 * bond_opt_parse - parse option value
 * @opt: the option to parse against
 * @val: value to parse
 *
 * This function tries to extract the value from @val and check if it's
 * a possible match for the option and returns NULL if a match isn't found,
 * or the struct_opt_value that matched. It also strips the new line from
 * @val->string if it's present.
 */
const struct bond_opt_value *bond_opt_parse(const struct bond_option *opt,
                                            struct bond_opt_value *val)
{
        char *p, valstr[BOND_OPT_MAX_NAMELEN + 1] = { 0, };
        const struct bond_opt_value *tbl;
        const struct bond_opt_value *ret = NULL;
        bool checkval;
        int i, rv;

        /* No parsing if the option wants a raw val */
        if (opt->flags & BOND_OPTFLAG_RAWVAL)
                return val;

        tbl = opt->values;
        if (!tbl)
                goto out;

        /* ULLONG_MAX is used to bypass string processing */
        checkval = val->value != ULLONG_MAX;
        if (!checkval) {
                if (!val->string)
                        goto out;
                p = strchr(val->string, '\n');
                if (p)
                        *p = '\0';
                for (p = val->string; *p; p++)
                        if (!(isdigit(*p) || isspace(*p)))
                                break;
                /* The following code extracts the string to match or the value
                 * and sets checkval appropriately
                 */
                if (*p) {
                        rv = sscanf(val->string, "%32s", valstr);
                } else {
                        rv = sscanf(val->string, "%llu", &val->value);
                        checkval = true;
                }
                if (!rv)
                        goto out;
        }

        for (i = 0; tbl[i].string; i++) {
                /* Check for exact match */
                if (checkval) {
                        if (val->value == tbl[i].value)
                                ret = &tbl[i];
                } else {
                        if (!strcmp(valstr, "default") &&
                            (tbl[i].flags & BOND_VALFLAG_DEFAULT))
                                ret = &tbl[i];

                        if (!strcmp(valstr, tbl[i].string))
                                ret = &tbl[i];
                }
                /* Found an exact match */
                if (ret)
                        goto out;
        }
        /* Possible range match */
        if (checkval && bond_opt_check_range(opt, val->value))
                ret = val;
out:
        return ret;
}

/* Check opt's dependencies against bond mode and currently set options */
static int bond_opt_check_deps(struct bonding *bond,
                               const struct bond_option *opt)
{
        struct bond_params *params = &bond->params;

        if (test_bit(params->mode, &opt->unsuppmodes))
                return -EACCES;
        if ((opt->flags & BOND_OPTFLAG_NOSLAVES) && bond_has_slaves(bond))
                return -ENOTEMPTY;
        if ((opt->flags & BOND_OPTFLAG_IFDOWN) && (bond->dev->flags & IFF_UP))
                return -EBUSY;

        return 0;
}

static void bond_opt_dep_print(struct bonding *bond,
                               const struct bond_option *opt,
                               struct nlattr *bad_attr,
                               struct netlink_ext_ack *extack)
{
        const struct bond_opt_value *modeval;
        struct bond_params *params;

        params = &bond->params;
        modeval = bond_opt_get_val(BOND_OPT_MODE, params->mode);
        if (test_bit(params->mode, &opt->unsuppmodes)) {
                netdev_err(bond->dev, "option %s: mode dependency failed, not supported in mode %s(%llu)\n",
                           opt->name, modeval->string, modeval->value);
                NL_SET_ERR_MSG_ATTR(extack, bad_attr,
                                    "option not supported in mode");
        }
}

static void bond_opt_error_interpret(struct bonding *bond,
                                     const struct bond_option *opt,
                                     int error, const struct bond_opt_value *val,
                                     struct nlattr *bad_attr,
                                     struct netlink_ext_ack *extack)
{
        const struct bond_opt_value *minval, *maxval;
        char *p;

        switch (error) {
        case -EINVAL:
                NL_SET_ERR_MSG_ATTR(extack, bad_attr, "invalid option value");
                if (val) {
                        if (val->string) {
                                /* sometimes RAWVAL opts may have new lines */
                                p = strchr(val->string, '\n');
                                if (p)
                                        *p = '\0';
                                netdev_err(bond->dev, "option %s: invalid value (%s)\n",
                                           opt->name, val->string);
                        } else {
                                netdev_err(bond->dev, "option %s: invalid value (%llu)\n",
                                           opt->name, val->value);
                        }
                }
                minval = bond_opt_get_flags(opt, BOND_VALFLAG_MIN);
                maxval = bond_opt_get_flags(opt, BOND_VALFLAG_MAX);
                if (!maxval)
                        break;
                netdev_err(bond->dev, "option %s: allowed values %llu - %llu\n",
                           opt->name, minval ? minval->value : 0, maxval->value);
                break;
        case -EACCES:
                bond_opt_dep_print(bond, opt, bad_attr, extack);
                break;
        case -ENOTEMPTY:
                NL_SET_ERR_MSG_ATTR(extack, bad_attr,
                                    "unable to set option because the bond device has slaves");
                netdev_err(bond->dev, "option %s: unable to set because the bond device has slaves\n",
                           opt->name);
                break;
        case -EBUSY:
                NL_SET_ERR_MSG_ATTR(extack, bad_attr,
                                    "unable to set option because the bond is up");
                netdev_err(bond->dev, "option %s: unable to set because the bond device is up\n",
                           opt->name);
                break;
        case -ENODEV:
                if (val && val->string) {
                        p = strchr(val->string, '\n');
                        if (p)
                                *p = '\0';
                        netdev_err(bond->dev, "option %s: interface %s does not exist!\n",
                                   opt->name, val->string);
                        NL_SET_ERR_MSG_ATTR(extack, bad_attr,
                                            "interface does not exist");
                }
                break;
        default:
                break;
        }
}

/**
 * __bond_opt_set - set a bonding option
 * @bond: target bond device
 * @option: option to set
 * @val: value to set it to
 * @bad_attr: netlink attribue that caused the error
 * @extack: extended netlink error structure, used when an error message
 *          needs to be returned to the caller via netlink
 *
 * This function is used to change the bond's option value, it can be
 * used for both enabling/changing an option and for disabling it. RTNL lock
 * must be obtained before calling this function.
 */
int __bond_opt_set(struct bonding *bond,
                   unsigned int option, struct bond_opt_value *val,
                   struct nlattr *bad_attr, struct netlink_ext_ack *extack)
{
        const struct bond_opt_value *retval = NULL;
        const struct bond_option *opt;
        int ret = -ENOENT;

        ASSERT_RTNL();

        opt = bond_opt_get(option);
        if (WARN_ON(!val) || WARN_ON(!opt))
                goto out;
        ret = bond_opt_check_deps(bond, opt);
        if (ret)
                goto out;
        retval = bond_opt_parse(opt, val);
        if (!retval) {
                ret = -EINVAL;
                goto out;
        }
        ret = opt->set(bond, retval);
out:
        if (ret)
                bond_opt_error_interpret(bond, opt, ret, val, bad_attr, extack);

        return ret;
}
/**
 * __bond_opt_set_notify - set a bonding option
 * @bond: target bond device
 * @option: option to set
 * @val: value to set it to
 *
 * This function is used to change the bond's option value and trigger
 * a notification to user sapce. It can be used for both enabling/changing
 * an option and for disabling it. RTNL lock must be obtained before calling
 * this function.
 */
int __bond_opt_set_notify(struct bonding *bond,
                          unsigned int option, struct bond_opt_value *val)
{
        int ret;

        ASSERT_RTNL();

        ret = __bond_opt_set(bond, option, val, NULL, NULL);

        if (!ret && (bond->dev->reg_state == NETREG_REGISTERED))
                call_netdevice_notifiers(NETDEV_CHANGEINFODATA, bond->dev);

        return ret;
}

/**
 * bond_opt_tryset_rtnl - try to acquire rtnl and call __bond_opt_set
 * @bond: target bond device
 * @option: option to set
 * @buf: value to set it to
 *
 * This function tries to acquire RTNL without blocking and if successful
 * calls __bond_opt_set. It is mainly used for sysfs option manipulation.
 */
int bond_opt_tryset_rtnl(struct bonding *bond, unsigned int option, char *buf)
{
        struct bond_opt_value optval;
        int ret;

        if (!rtnl_trylock())
                return restart_syscall();
        bond_opt_initstr(&optval, buf);
        ret = __bond_opt_set_notify(bond, option, &optval);
        rtnl_unlock();

        return ret;
}

/**
 * bond_opt_get - get a pointer to an option
 * @option: option for which to return a pointer
 *
 * This function checks if option is valid and if so returns a pointer
 * to its entry in the bond_opts[] option array.
 */
const struct bond_option *bond_opt_get(unsigned int option)
{
        if (!BOND_OPT_VALID(option))
                return NULL;

        return &bond_opts[option];
}

static bool bond_set_xfrm_features(struct bonding *bond)
{
        if (!IS_ENABLED(CONFIG_XFRM_OFFLOAD))
                return false;

        if (BOND_MODE(bond) == BOND_MODE_ACTIVEBACKUP)
                bond->dev->wanted_features |= BOND_XFRM_FEATURES;
        else
                bond->dev->wanted_features &= ~BOND_XFRM_FEATURES;

        return true;
}

static int bond_option_mode_set(struct bonding *bond,
                                const struct bond_opt_value *newval)
{
        if (bond->xdp_prog && !bond_xdp_check(bond, newval->value))
                return -EOPNOTSUPP;

        if (!bond_mode_uses_arp(newval->value)) {
                if (bond->params.arp_interval) {
                        netdev_dbg(bond->dev, "%s mode is incompatible with arp monitoring, start mii monitoring\n",
                                   newval->string);
                        /* disable arp monitoring */
                        bond->params.arp_interval = 0;
                }

                if (!bond->params.miimon) {
                        /* set miimon to default value */
                        bond->params.miimon = BOND_DEFAULT_MIIMON;
                        netdev_dbg(bond->dev, "Setting MII monitoring interval to %d\n",
                                   bond->params.miimon);
                }
        }

        if (newval->value == BOND_MODE_ALB)
                bond->params.tlb_dynamic_lb = 1;

        /* don't cache arp_validate between modes */
        bond->params.arp_validate = BOND_ARP_VALIDATE_NONE;
        bond->params.mode = newval->value;

        if (bond->dev->reg_state == NETREG_REGISTERED) {
                bool update = false;

                update |= bond_set_xfrm_features(bond);

                if (update)
                        netdev_update_features(bond->dev);
        }

        bond_xdp_set_features(bond->dev);

        return 0;
}

static int bond_option_active_slave_set(struct bonding *bond,
                                        const struct bond_opt_value *newval)
{
        char ifname[IFNAMSIZ] = { 0, };
        struct net_device *slave_dev;
        int ret = 0;

        sscanf(newval->string, "%15s", ifname); /* IFNAMSIZ */
        if (!strlen(ifname) || newval->string[0] == '\n') {
                slave_dev = NULL;
        } else {
                slave_dev = __dev_get_by_name(dev_net(bond->dev), ifname);
                if (!slave_dev)
                        return -ENODEV;
        }

        if (slave_dev) {
                if (!netif_is_bond_slave(slave_dev)) {
                        slave_err(bond->dev, slave_dev, "Device is not bonding slave\n");
                        return -EINVAL;
                }

                if (bond->dev != netdev_master_upper_dev_get(slave_dev)) {
                        slave_err(bond->dev, slave_dev, "Device is not our slave\n");
                        return -EINVAL;
                }
        }

        block_netpoll_tx();
        /* check to see if we are clearing active */
        if (!slave_dev) {
                netdev_dbg(bond->dev, "Clearing current active slave\n");
                bond_change_active_slave(bond, NULL);
                bond_select_active_slave(bond);
        } else {
                struct slave *old_active = rtnl_dereference(bond->curr_active_slave);
                struct slave *new_active = bond_slave_get_rtnl(slave_dev);

                BUG_ON(!new_active);

                if (new_active == old_active) {
                        /* do nothing */
                        slave_dbg(bond->dev, new_active->dev, "is already the current active slave\n");
                } else {
                        if (old_active && (new_active->link == BOND_LINK_UP) &&
                            bond_slave_is_up(new_active)) {
                                slave_dbg(bond->dev, new_active->dev, "Setting as active slave\n");
                                bond_change_active_slave(bond, new_active);
                        } else {
                                slave_err(bond->dev, new_active->dev, "Could not set as active slave; either %s is down or the link is down\n",
                                          new_active->dev->name);
                                ret = -EINVAL;
                        }
                }
        }
        unblock_netpoll_tx();

        return ret;
}

/* There are two tricky bits here.  First, if MII monitoring is activated, then
 * we must disable ARP monitoring.  Second, if the timer isn't running, we must
 * start it.
 */
static int bond_option_miimon_set(struct bonding *bond,
                                  const struct bond_opt_value *newval)
{
        netdev_dbg(bond->dev, "Setting MII monitoring interval to %llu\n",
                   newval->value);
        bond->params.miimon = newval->value;
        if (bond->params.updelay)
                netdev_dbg(bond->dev, "Note: Updating updelay (to %d) since it is a multiple of the miimon value\n",
                           bond->params.updelay * bond->params.miimon);
        if (bond->params.downdelay)
                netdev_dbg(bond->dev, "Note: Updating downdelay (to %d) since it is a multiple of the miimon value\n",
                           bond->params.downdelay * bond->params.miimon);
        if (bond->params.peer_notif_delay)
                netdev_dbg(bond->dev, "Note: Updating peer_notif_delay (to %d) since it is a multiple of the miimon value\n",
                           bond->params.peer_notif_delay * bond->params.miimon);
        if (newval->value && bond->params.arp_interval) {
                netdev_dbg(bond->dev, "MII monitoring cannot be used with ARP monitoring - disabling ARP monitoring...\n");
                bond->params.arp_interval = 0;
                if (bond->params.arp_validate)
                        bond->params.arp_validate = BOND_ARP_VALIDATE_NONE;
        }
        if (bond->dev->flags & IFF_UP) {
                /* If the interface is up, we may need to fire off
                 * the MII timer. If the interface is down, the
                 * timer will get fired off when the open function
                 * is called.
                 */
                if (!newval->value) {
                        cancel_delayed_work_sync(&bond->mii_work);
                } else {
                        cancel_delayed_work_sync(&bond->arp_work);
                        queue_delayed_work(bond->wq, &bond->mii_work, 0);
                }
        }

        return 0;
}

/* Set up, down and peer notification delays. These must be multiples
 * of the MII monitoring value, and are stored internally as the
 * multiplier. Thus, we must translate to MS for the real world.
 */
static int _bond_option_delay_set(struct bonding *bond,
                                  const struct bond_opt_value *newval,
                                  const char *name,
                                  int *target)
{
        int value = newval->value;

        if (!bond->params.miimon) {
                netdev_err(bond->dev, "Unable to set %s as MII monitoring is disabled\n",
                           name);
                return -EPERM;
        }
        if ((value % bond->params.miimon) != 0) {
                netdev_warn(bond->dev,
                            "%s (%d) is not a multiple of miimon (%d), value rounded to %d ms\n",
                            name,
                            value, bond->params.miimon,
                            (value / bond->params.miimon) *
                            bond->params.miimon);
        }
        *target = value / bond->params.miimon;
        netdev_dbg(bond->dev, "Setting %s to %d\n",
                   name,
                   *target * bond->params.miimon);

        return 0;
}

static int bond_option_updelay_set(struct bonding *bond,
                                   const struct bond_opt_value *newval)
{
        return _bond_option_delay_set(bond, newval, "up delay",
                                      &bond->params.updelay);
}

static int bond_option_downdelay_set(struct bonding *bond,
                                     const struct bond_opt_value *newval)
{
        return _bond_option_delay_set(bond, newval, "down delay",
                                      &bond->params.downdelay);
}

static int bond_option_peer_notif_delay_set(struct bonding *bond,
                                            const struct bond_opt_value *newval)
{
        int ret = _bond_option_delay_set(bond, newval,
                                         "peer notification delay",
                                         &bond->params.peer_notif_delay);
        return ret;
}

static int bond_option_use_carrier_set(struct bonding *bond,
                                       const struct bond_opt_value *newval)
{
        netdev_dbg(bond->dev, "Setting use_carrier to %llu\n",
                   newval->value);
        bond->params.use_carrier = newval->value;

        return 0;
}

/* There are two tricky bits here.  First, if ARP monitoring is activated, then
 * we must disable MII monitoring.  Second, if the ARP timer isn't running,
 * we must start it.
 */
static int bond_option_arp_interval_set(struct bonding *bond,
                                        const struct bond_opt_value *newval)
{
        netdev_dbg(bond->dev, "Setting ARP monitoring interval to %llu\n",
                   newval->value);
        bond->params.arp_interval = newval->value;
        if (newval->value) {
                if (bond->params.miimon) {
                        netdev_dbg(bond->dev, "ARP monitoring cannot be used with MII monitoring. Disabling MII monitoring\n");
                        bond->params.miimon = 0;
                }
                if (!bond->params.arp_targets[0])
                        netdev_dbg(bond->dev, "ARP monitoring has been set up, but no ARP targets have been specified\n");
        }
        if (bond->dev->flags & IFF_UP) {
                /* If the interface is up, we may need to fire off
                 * the ARP timer.  If the interface is down, the
                 * timer will get fired off when the open function
                 * is called.
                 */
                if (!newval->value) {
                        if (bond->params.arp_validate)
                                bond->recv_probe = NULL;
                        cancel_delayed_work_sync(&bond->arp_work);
                } else {
                        /* arp_validate can be set only in active-backup mode */
                        bond->recv_probe = bond_rcv_validate;
                        cancel_delayed_work_sync(&bond->mii_work);
                        queue_delayed_work(bond->wq, &bond->arp_work, 0);
                }
        }

        return 0;
}

static void _bond_options_arp_ip_target_set(struct bonding *bond, int slot,
                                            __be32 target,
                                            unsigned long last_rx)
{
        __be32 *targets = bond->params.arp_targets;
        struct list_head *iter;
        struct slave *slave;

        if (slot >= 0 && slot < BOND_MAX_ARP_TARGETS) {
                bond_for_each_slave(bond, slave, iter)
                        slave->target_last_arp_rx[slot] = last_rx;
                targets[slot] = target;
        }
}

static int _bond_option_arp_ip_target_add(struct bonding *bond, __be32 target)
{
        __be32 *targets = bond->params.arp_targets;
        int ind;

        if (!bond_is_ip_target_ok(target)) {
                netdev_err(bond->dev, "invalid ARP target %pI4 specified for addition\n",
                           &target);
                return -EINVAL;
        }

        if (bond_get_targets_ip(targets, target) != -1) { /* dup */
                netdev_err(bond->dev, "ARP target %pI4 is already present\n",
                           &target);
                return -EINVAL;
        }

        ind = bond_get_targets_ip(targets, 0); /* first free slot */
        if (ind == -1) {
                netdev_err(bond->dev, "ARP target table is full!\n");
                return -EINVAL;
        }

        netdev_dbg(bond->dev, "Adding ARP target %pI4\n", &target);

        _bond_options_arp_ip_target_set(bond, ind, target, jiffies);

        return 0;
}

static int bond_option_arp_ip_target_add(struct bonding *bond, __be32 target)
{
        return _bond_option_arp_ip_target_add(bond, target);
}

static int bond_option_arp_ip_target_rem(struct bonding *bond, __be32 target)
{
        __be32 *targets = bond->params.arp_targets;
        struct list_head *iter;
        struct slave *slave;
        unsigned long *targets_rx;
        int ind, i;

        if (!bond_is_ip_target_ok(target)) {
                netdev_err(bond->dev, "invalid ARP target %pI4 specified for removal\n",
                           &target);
                return -EINVAL;
        }

        ind = bond_get_targets_ip(targets, target);
        if (ind == -1) {
                netdev_err(bond->dev, "unable to remove nonexistent ARP target %pI4\n",
                           &target);
                return -EINVAL;
        }

        if (ind == 0 && !targets[1] && bond->params.arp_interval)
                netdev_warn(bond->dev, "Removing last arp target with arp_interval on\n");

        netdev_dbg(bond->dev, "Removing ARP target %pI4\n", &target);

        bond_for_each_slave(bond, slave, iter) {
                targets_rx = slave->target_last_arp_rx;
                for (i = ind; (i < BOND_MAX_ARP_TARGETS-1) && targets[i+1]; i++)
                        targets_rx[i] = targets_rx[i+1];
                targets_rx[i] = 0;
        }
        for (i = ind; (i < BOND_MAX_ARP_TARGETS-1) && targets[i+1]; i++)
                targets[i] = targets[i+1];
        targets[i] = 0;

        return 0;
}

void bond_option_arp_ip_targets_clear(struct bonding *bond)
{
        int i;

        for (i = 0; i < BOND_MAX_ARP_TARGETS; i++)
                _bond_options_arp_ip_target_set(bond, i, 0, 0);
}

static int bond_option_arp_ip_targets_set(struct bonding *bond,
                                          const struct bond_opt_value *newval)
{
        int ret = -EPERM;
        __be32 target;

        if (newval->string) {
                if (strlen(newval->string) < 1 ||
                    !in4_pton(newval->string + 1, -1, (u8 *)&target, -1, NULL)) {
                        netdev_err(bond->dev, "invalid ARP target specified\n");
                        return ret;
                }
                if (newval->string[0] == '+')
                        ret = bond_option_arp_ip_target_add(bond, target);
                else if (newval->string[0] == '-')
                        ret = bond_option_arp_ip_target_rem(bond, target);
                else
                        netdev_err(bond->dev, "no command found in arp_ip_targets file - use +<addr> or -<addr>\n");
        } else {
                target = newval->value;
                ret = bond_option_arp_ip_target_add(bond, target);
        }

        return ret;
}

#if IS_ENABLED(CONFIG_IPV6)
static bool slave_can_set_ns_maddr(const struct bonding *bond, struct slave *slave)
{
        return BOND_MODE(bond) == BOND_MODE_ACTIVEBACKUP &&
               !bond_is_active_slave(slave) &&
               slave->dev->flags & IFF_MULTICAST;
}

/**
 * slave_set_ns_maddrs - add/del all NS mac addresses for slave
 * @bond: bond device
 * @slave: slave device
 * @add: add or remove all the NS mac addresses
 *
 * This function tries to add or delete all the NS mac addresses on the slave
 *
 * Note, the IPv6 NS target address is the unicast address in Neighbor
 * Solicitation (NS) message. The dest address of NS message should be
 * solicited-node multicast address of the target. The dest mac of NS message
 * is converted from the solicited-node multicast address.
 *
 * This function is called when
 *   * arp_validate changes
 *   * enslaving, releasing new slaves
 */
static void slave_set_ns_maddrs(struct bonding *bond, struct slave *slave, bool add)
{
        struct in6_addr *targets = bond->params.ns_targets;
        char slot_maddr[MAX_ADDR_LEN];
        struct in6_addr mcaddr;
        int i;

        if (!slave_can_set_ns_maddr(bond, slave))
                return;

        for (i = 0; i < BOND_MAX_NS_TARGETS; i++) {
                if (ipv6_addr_any(&targets[i]))
                        break;

                addrconf_addr_solict_mult(&targets[i], &mcaddr);
                if (!ndisc_mc_map(&mcaddr, slot_maddr, slave->dev, 0)) {
                        if (add)
                                dev_mc_add(slave->dev, slot_maddr);
                        else
                                dev_mc_del(slave->dev, slot_maddr);
                }
        }
}

void bond_slave_ns_maddrs_add(struct bonding *bond, struct slave *slave)
{
        if (!bond->params.arp_validate)
                return;
        slave_set_ns_maddrs(bond, slave, true);
}

void bond_slave_ns_maddrs_del(struct bonding *bond, struct slave *slave)
{
        if (!bond->params.arp_validate)
                return;
        slave_set_ns_maddrs(bond, slave, false);
}

/**
 * slave_set_ns_maddr - set new NS mac address for slave
 * @bond: bond device
 * @slave: slave device
 * @target: the new IPv6 target
 * @slot: the old IPv6 target in the slot
 *
 * This function tries to replace the old mac address to new one on the slave.
 *
 * Note, the target/slot IPv6 address is the unicast address in Neighbor
 * Solicitation (NS) message. The dest address of NS message should be
 * solicited-node multicast address of the target. The dest mac of NS message
 * is converted from the solicited-node multicast address.
 *
 * This function is called when
 *   * An IPv6 NS target is added or removed.
 */
static void slave_set_ns_maddr(struct bonding *bond, struct slave *slave,
                               struct in6_addr *target, struct in6_addr *slot)
{
        char mac_addr[MAX_ADDR_LEN];
        struct in6_addr mcast_addr;

        if (!bond->params.arp_validate || !slave_can_set_ns_maddr(bond, slave))
                return;

        /* remove the previous mac addr from slave */
        addrconf_addr_solict_mult(slot, &mcast_addr);
        if (!ipv6_addr_any(slot) &&
            !ndisc_mc_map(&mcast_addr, mac_addr, slave->dev, 0))
                dev_mc_del(slave->dev, mac_addr);

        /* add new mac addr on slave if target is set */
        addrconf_addr_solict_mult(target, &mcast_addr);
        if (!ipv6_addr_any(target) &&
            !ndisc_mc_map(&mcast_addr, mac_addr, slave->dev, 0))
                dev_mc_add(slave->dev, mac_addr);
}

static void _bond_options_ns_ip6_target_set(struct bonding *bond, int slot,
                                            struct in6_addr *target,
                                            unsigned long last_rx)
{
        struct in6_addr *targets = bond->params.ns_targets;
        struct list_head *iter;
        struct slave *slave;

        if (slot >= 0 && slot < BOND_MAX_NS_TARGETS) {
                bond_for_each_slave(bond, slave, iter) {
                        slave->target_last_arp_rx[slot] = last_rx;
                        slave_set_ns_maddr(bond, slave, target, &targets[slot]);
                }
                targets[slot] = *target;
        }
}

void bond_option_ns_ip6_targets_clear(struct bonding *bond)
{
        struct in6_addr addr_any = in6addr_any;
        int i;

        for (i = 0; i < BOND_MAX_NS_TARGETS; i++)
                _bond_options_ns_ip6_target_set(bond, i, &addr_any, 0);
}

static int bond_option_ns_ip6_targets_set(struct bonding *bond,
                                          const struct bond_opt_value *newval)
{
        struct in6_addr *target = (struct in6_addr *)newval->extra;
        struct in6_addr *targets = bond->params.ns_targets;
        struct in6_addr addr_any = in6addr_any;
        int index;

        if (!bond_is_ip6_target_ok(target)) {
                netdev_err(bond->dev, "invalid NS target %pI6c specified for addition\n",
                           target);
                return -EINVAL;
        }

        if (bond_get_targets_ip6(targets, target) != -1) { /* dup */
                netdev_err(bond->dev, "NS target %pI6c is already present\n",
                           target);
                return -EINVAL;
        }

        index = bond_get_targets_ip6(targets, &addr_any); /* first free slot */
        if (index == -1) {
                netdev_err(bond->dev, "NS target table is full!\n");
                return -EINVAL;
        }

        netdev_dbg(bond->dev, "Adding NS target %pI6c\n", target);

        _bond_options_ns_ip6_target_set(bond, index, target, jiffies);

        return 0;
}
#else
static int bond_option_ns_ip6_targets_set(struct bonding *bond,
                                          const struct bond_opt_value *newval)
{
        return -EPERM;
}

static void slave_set_ns_maddrs(struct bonding *bond, struct slave *slave, bool add) {}

void bond_slave_ns_maddrs_add(struct bonding *bond, struct slave *slave) {}

void bond_slave_ns_maddrs_del(struct bonding *bond, struct slave *slave) {}
#endif

static int bond_option_arp_validate_set(struct bonding *bond,
                                        const struct bond_opt_value *newval)
{
        bool changed = !!bond->params.arp_validate != !!newval->value;
        struct list_head *iter;
        struct slave *slave;

        netdev_dbg(bond->dev, "Setting arp_validate to %s (%llu)\n",
                   newval->string, newval->value);
        bond->params.arp_validate = newval->value;

        if (changed) {
                bond_for_each_slave(bond, slave, iter)
                        slave_set_ns_maddrs(bond, slave, !!bond->params.arp_validate);
        }

        return 0;
}

static int bond_option_arp_all_targets_set(struct bonding *bond,
                                           const struct bond_opt_value *newval)
{
        netdev_dbg(bond->dev, "Setting arp_all_targets to %s (%llu)\n",
                   newval->string, newval->value);
        bond->params.arp_all_targets = newval->value;

        return 0;
}

static int bond_option_missed_max_set(struct bonding *bond,
                                      const struct bond_opt_value *newval)
{
        netdev_dbg(bond->dev, "Setting missed max to %s (%llu)\n",
                   newval->string, newval->value);
        bond->params.missed_max = newval->value;

        return 0;
}

static int bond_option_prio_set(struct bonding *bond,
                                const struct bond_opt_value *newval)
{
        struct slave *slave;

        slave = bond_slave_get_rtnl(newval->slave_dev);
        if (!slave) {
                netdev_dbg(newval->slave_dev, "%s called on NULL slave\n", __func__);
                return -ENODEV;
        }
        slave->prio = newval->value;

        if (rtnl_dereference(bond->primary_slave))
                slave_warn(bond->dev, slave->dev,
                           "prio updated, but will not affect failover re-selection as primary slave have been set\n");
        else
                bond_select_active_slave(bond);

        return 0;
}

static int bond_option_primary_set(struct bonding *bond,
                                   const struct bond_opt_value *newval)
{
        char *p, *primary = newval->string;
        struct list_head *iter;
        struct slave *slave;

        block_netpoll_tx();

        p = strchr(primary, '\n');
        if (p)
                *p = '\0';
        /* check to see if we are clearing primary */
        if (!strlen(primary)) {
                netdev_dbg(bond->dev, "Setting primary slave to None\n");
                RCU_INIT_POINTER(bond->primary_slave, NULL);
                memset(bond->params.primary, 0, sizeof(bond->params.primary));
                bond_select_active_slave(bond);
                goto out;
        }

        bond_for_each_slave(bond, slave, iter) {
                if (strncmp(slave->dev->name, primary, IFNAMSIZ) == 0) {
                        slave_dbg(bond->dev, slave->dev, "Setting as primary slave\n");
                        rcu_assign_pointer(bond->primary_slave, slave);
                        strcpy(bond->params.primary, slave->dev->name);
                        bond->force_primary = true;
                        bond_select_active_slave(bond);
                        goto out;
                }
        }

        if (rtnl_dereference(bond->primary_slave)) {
                netdev_dbg(bond->dev, "Setting primary slave to None\n");
                RCU_INIT_POINTER(bond->primary_slave, NULL);
                bond_select_active_slave(bond);
        }
        strscpy_pad(bond->params.primary, primary, IFNAMSIZ);

        netdev_dbg(bond->dev, "Recording %s as primary, but it has not been enslaved yet\n",
                   primary);

out:
        unblock_netpoll_tx();

        return 0;
}

static int bond_option_primary_reselect_set(struct bonding *bond,
                                            const struct bond_opt_value *newval)
{
        netdev_dbg(bond->dev, "Setting primary_reselect to %s (%llu)\n",
                   newval->string, newval->value);
        bond->params.primary_reselect = newval->value;

        block_netpoll_tx();
        bond_select_active_slave(bond);
        unblock_netpoll_tx();

        return 0;
}

static int bond_option_fail_over_mac_set(struct bonding *bond,
                                         const struct bond_opt_value *newval)
{
        netdev_dbg(bond->dev, "Setting fail_over_mac to %s (%llu)\n",
                   newval->string, newval->value);
        bond->params.fail_over_mac = newval->value;

        return 0;
}

static int bond_option_xmit_hash_policy_set(struct bonding *bond,
                                            const struct bond_opt_value *newval)
{
        netdev_dbg(bond->dev, "Setting xmit hash policy to %s (%llu)\n",
                   newval->string, newval->value);
        bond->params.xmit_policy = newval->value;

        return 0;
}

static int bond_option_resend_igmp_set(struct bonding *bond,
                                       const struct bond_opt_value *newval)
{
        netdev_dbg(bond->dev, "Setting resend_igmp to %llu\n",
                   newval->value);
        bond->params.resend_igmp = newval->value;

        return 0;
}

static int bond_option_num_peer_notif_set(struct bonding *bond,
                                   const struct bond_opt_value *newval)
{
        bond->params.num_peer_notif = newval->value;

        return 0;
}

static int bond_option_all_slaves_active_set(struct bonding *bond,
                                             const struct bond_opt_value *newval)
{
        struct list_head *iter;
        struct slave *slave;

        if (newval->value == bond->params.all_slaves_active)
                return 0;
        bond->params.all_slaves_active = newval->value;
        bond_for_each_slave(bond, slave, iter) {
                if (!bond_is_active_slave(slave)) {
                        if (newval->value)
                                slave->inactive = 0;
                        else
                                slave->inactive = 1;
                }
        }

        return 0;
}

static int bond_option_min_links_set(struct bonding *bond,
                                     const struct bond_opt_value *newval)
{
        netdev_dbg(bond->dev, "Setting min links value to %llu\n",
                   newval->value);
        bond->params.min_links = newval->value;
        bond_set_carrier(bond);

        return 0;
}

static int bond_option_lp_interval_set(struct bonding *bond,
                                       const struct bond_opt_value *newval)
{
        bond->params.lp_interval = newval->value;

        return 0;
}

static int bond_option_pps_set(struct bonding *bond,
                               const struct bond_opt_value *newval)
{
        netdev_dbg(bond->dev, "Setting packets per slave to %llu\n",
                   newval->value);
        bond->params.packets_per_slave = newval->value;
        if (newval->value > 0) {
                bond->params.reciprocal_packets_per_slave =
                        reciprocal_value(newval->value);
        } else {
                /* reciprocal_packets_per_slave is unused if
                 * packets_per_slave is 0 or 1, just initialize it
                 */
                bond->params.reciprocal_packets_per_slave =
                        (struct reciprocal_value) { 0 };
        }

        return 0;
}

static int bond_option_lacp_active_set(struct bonding *bond,
                                       const struct bond_opt_value *newval)
{
        netdev_dbg(bond->dev, "Setting LACP active to %s (%llu)\n",
                   newval->string, newval->value);
        bond->params.lacp_active = newval->value;

        return 0;
}

static int bond_option_lacp_rate_set(struct bonding *bond,
                                     const struct bond_opt_value *newval)
{
        netdev_dbg(bond->dev, "Setting LACP rate to %s (%llu)\n",
                   newval->string, newval->value);
        bond->params.lacp_fast = newval->value;
        bond_3ad_update_lacp_rate(bond);

        return 0;
}

static int bond_option_ad_select_set(struct bonding *bond,
                                     const struct bond_opt_value *newval)
{
        netdev_dbg(bond->dev, "Setting ad_select to %s (%llu)\n",
                   newval->string, newval->value);
        bond->params.ad_select = newval->value;

        return 0;
}

static int bond_option_queue_id_set(struct bonding *bond,
                                    const struct bond_opt_value *newval)
{
        struct slave *slave, *update_slave;
        struct net_device *sdev;
        struct list_head *iter;
        char *delim;
        int ret = 0;
        u16 qid;

        /* delim will point to queue id if successful */
        delim = strchr(newval->string, ':');
        if (!delim)
                goto err_no_cmd;

        /* Terminate string that points to device name and bump it
         * up one, so we can read the queue id there.
         */
        *delim = '\0';
        if (sscanf(++delim, "%hd\n", &qid) != 1)
                goto err_no_cmd;

        /* Check buffer length, valid ifname and queue id */
        if (!dev_valid_name(newval->string) ||
            qid > bond->dev->real_num_tx_queues)
                goto err_no_cmd;

        /* Get the pointer to that interface if it exists */
        sdev = __dev_get_by_name(dev_net(bond->dev), newval->string);
        if (!sdev)
                goto err_no_cmd;

        /* Search for thes slave and check for duplicate qids */
        update_slave = NULL;
        bond_for_each_slave(bond, slave, iter) {
                if (sdev == slave->dev)
                        /* We don't need to check the matching
                         * slave for dups, since we're overwriting it
                         */
                        update_slave = slave;
                else if (qid && qid == slave->queue_id) {
                        goto err_no_cmd;
                }
        }

        if (!update_slave)
                goto err_no_cmd;

        /* Actually set the qids for the slave */
        WRITE_ONCE(update_slave->queue_id, qid);

out:
        return ret;

err_no_cmd:
        netdev_dbg(bond->dev, "invalid input for queue_id set\n");
        ret = -EPERM;
        goto out;

}

static int bond_option_slaves_set(struct bonding *bond,
                                  const struct bond_opt_value *newval)
{
        char command[IFNAMSIZ + 1] = { 0, };
        struct net_device *dev;
        char *ifname;
        int ret;

        sscanf(newval->string, "%16s", command); /* IFNAMSIZ*/
        ifname = command + 1;
        if ((strlen(command) <= 1) ||
            (command[0] != '+' && command[0] != '-') ||
            !dev_valid_name(ifname))
                goto err_no_cmd;

        dev = __dev_get_by_name(dev_net(bond->dev), ifname);
        if (!dev) {
                netdev_dbg(bond->dev, "interface %s does not exist!\n",
                           ifname);
                ret = -ENODEV;
                goto out;
        }

        switch (command[0]) {
        case '+':
                slave_dbg(bond->dev, dev, "Enslaving interface\n");
                ret = bond_enslave(bond->dev, dev, NULL);
                break;

        case '-':
                slave_dbg(bond->dev, dev, "Releasing interface\n");
                ret = bond_release(bond->dev, dev);
                break;

        default:
                /* should not run here. */
                goto err_no_cmd;
        }

out:
        return ret;

err_no_cmd:
        netdev_err(bond->dev, "no command found in slaves file - use +ifname or -ifname\n");
        ret = -EPERM;
        goto out;
}

static int bond_option_tlb_dynamic_lb_set(struct bonding *bond,
                                          const struct bond_opt_value *newval)
{
        netdev_dbg(bond->dev, "Setting dynamic-lb to %s (%llu)\n",
                   newval->string, newval->value);
        bond->params.tlb_dynamic_lb = newval->value;

        return 0;
}

static int bond_option_ad_actor_sys_prio_set(struct bonding *bond,
                                             const struct bond_opt_value *newval)
{
        netdev_dbg(bond->dev, "Setting ad_actor_sys_prio to %llu\n",
                   newval->value);

        bond->params.ad_actor_sys_prio = newval->value;
        bond_3ad_update_ad_actor_settings(bond);

        return 0;
}

static int bond_option_ad_actor_system_set(struct bonding *bond,
                                           const struct bond_opt_value *newval)
{
        u8 macaddr[ETH_ALEN];
        u8 *mac;

        if (newval->string) {
                if (!mac_pton(newval->string, macaddr))
                        goto err;
                mac = macaddr;
        } else {
                mac = (u8 *)&newval->value;
        }

        if (is_multicast_ether_addr(mac))
                goto err;

        netdev_dbg(bond->dev, "Setting ad_actor_system to %pM\n", mac);
        ether_addr_copy(bond->params.ad_actor_system, mac);
        bond_3ad_update_ad_actor_settings(bond);

        return 0;

err:
        netdev_err(bond->dev, "Invalid ad_actor_system MAC address.\n");
        return -EINVAL;
}

static int bond_option_ad_user_port_key_set(struct bonding *bond,
                                            const struct bond_opt_value *newval)
{
        netdev_dbg(bond->dev, "Setting ad_user_port_key to %llu\n",
                   newval->value);

        bond->params.ad_user_port_key = newval->value;
        return 0;
}

static int bond_option_coupled_control_set(struct bonding *bond,
                                           const struct bond_opt_value *newval)
{
        netdev_info(bond->dev, "Setting coupled_control to %s (%llu)\n",
                    newval->string, newval->value);

        bond->params.coupled_control = newval->value;
        return 0;
}









































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_SPECIAL_INSNS_H
#define _ASM_X86_SPECIAL_INSNS_H

#ifdef __KERNEL__
#include <asm/nops.h>
#include <asm/processor-flags.h>

#include <linux/errno.h>
#include <linux/irqflags.h>
#include <linux/jump_label.h>

/*
 * The compiler should not reorder volatile asm statements with respect to each
 * other: they should execute in program order. However GCC 4.9.x and 5.x have
 * a bug (which was fixed in 8.1, 7.3 and 6.5) where they might reorder
 * volatile asm. The write functions are not affected since they have memory
 * clobbers preventing reordering. To prevent reads from being reordered with
 * respect to writes, use a dummy memory operand.
 */

#define __FORCE_ORDER "m"(*(unsigned int *)0x1000UL)

void native_write_cr0(unsigned long val);

static inline unsigned long native_read_cr0(void)
{
        unsigned long val;
        asm volatile("mov %%cr0,%0\n\t" : "=r" (val) : __FORCE_ORDER);
        return val;
}

static __always_inline unsigned long native_read_cr2(void)
{
        unsigned long val;
        asm volatile("mov %%cr2,%0\n\t" : "=r" (val) : __FORCE_ORDER);
        return val;
}

static __always_inline void native_write_cr2(unsigned long val)
{
        asm volatile("mov %0,%%cr2": : "r" (val) : "memory");
}

static __always_inline unsigned long __native_read_cr3(void)
{
        unsigned long val;
        asm volatile("mov %%cr3,%0\n\t" : "=r" (val) : __FORCE_ORDER);
        return val;
}

static __always_inline void native_write_cr3(unsigned long val)
{
        asm volatile("mov %0,%%cr3": : "r" (val) : "memory");
}

static inline unsigned long native_read_cr4(void)
{
        unsigned long val;
#ifdef CONFIG_X86_32
        /*
         * This could fault if CR4 does not exist.  Non-existent CR4
         * is functionally equivalent to CR4 == 0.  Keep it simple and pretend
         * that CR4 == 0 on CPUs that don't have CR4.
         */
        asm volatile("1: mov %%cr4, %0\n"
                     "2:\n"
                     _ASM_EXTABLE(1b, 2b)
                     : "=r" (val) : "0" (0), __FORCE_ORDER);
#else
        /* CR4 always exists on x86_64. */
        asm volatile("mov %%cr4,%0\n\t" : "=r" (val) : __FORCE_ORDER);
#endif
        return val;
}

void native_write_cr4(unsigned long val);

#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
static inline u32 rdpkru(void)
{
        u32 ecx = 0;
        u32 edx, pkru;

        /*
         * "rdpkru" instruction.  Places PKRU contents in to EAX,
         * clears EDX and requires that ecx=0.
         */
        asm volatile(".byte 0x0f,0x01,0xee\n\t"
                     : "=a" (pkru), "=d" (edx)
                     : "c" (ecx));
        return pkru;
}

static inline void wrpkru(u32 pkru)
{
        u32 ecx = 0, edx = 0;

        /*
         * "wrpkru" instruction.  Loads contents in EAX to PKRU,
         * requires that ecx = edx = 0.
         */
        asm volatile(".byte 0x0f,0x01,0xef\n\t"
                     : : "a" (pkru), "c"(ecx), "d"(edx));
}

#else
static inline u32 rdpkru(void)
{
        return 0;
}

static inline void wrpkru(u32 pkru)
{
}
#endif

static __always_inline void wbinvd(void)
{
        asm volatile("wbinvd": : :"memory");
}

static inline unsigned long __read_cr4(void)
{
        return native_read_cr4();
}

#ifdef CONFIG_PARAVIRT_XXL
#include <asm/paravirt.h>
#else

static inline unsigned long read_cr0(void)
{
        return native_read_cr0();
}

static inline void write_cr0(unsigned long x)
{
        native_write_cr0(x);
}

static __always_inline unsigned long read_cr2(void)
{
        return native_read_cr2();
}

static __always_inline void write_cr2(unsigned long x)
{
        native_write_cr2(x);
}

/*
 * Careful!  CR3 contains more than just an address.  You probably want
 * read_cr3_pa() instead.
 */
static inline unsigned long __read_cr3(void)
{
        return __native_read_cr3();
}

static inline void write_cr3(unsigned long x)
{
        native_write_cr3(x);
}

static inline void __write_cr4(unsigned long x)
{
        native_write_cr4(x);
}
#endif /* CONFIG_PARAVIRT_XXL */

static __always_inline void clflush(volatile void *__p)
{
        asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p));
}

static inline void clflushopt(volatile void *__p)
{
        alternative_io("ds clflush %0",
                       "clflushopt %0", X86_FEATURE_CLFLUSHOPT,
                       "+m" (*(volatile char __force *)__p));
}

static inline void clwb(volatile void *__p)
{
        volatile struct { char x[64]; } *p = __p;

        asm_inline volatile(ALTERNATIVE_2(
                "ds clflush %0",
                "clflushopt %0", X86_FEATURE_CLFLUSHOPT,
                "clwb %0", X86_FEATURE_CLWB)
                : "+m" (*p));
}

#ifdef CONFIG_X86_USER_SHADOW_STACK
static inline int write_user_shstk_64(u64 __user *addr, u64 val)
{
        asm goto("1: wrussq %[val], %[addr]\n"
                          _ASM_EXTABLE(1b, %l[fail])
                          :: [addr] "m" (*addr), [val] "r" (val)
                          :: fail);
        return 0;
fail:
        return -EFAULT;
}
#endif /* CONFIG_X86_USER_SHADOW_STACK */

#define nop() asm volatile ("nop")

static __always_inline void serialize(void)
{
        /* Instruction opcode for SERIALIZE; supported in binutils >= 2.35. */
        asm volatile(".byte 0xf, 0x1, 0xe8" ::: "memory");
}

/* The dst parameter must be 64-bytes aligned */
static inline void movdir64b(void *dst, const void *src)
{
        const struct { char _[64]; } *__src = src;
        struct { char _[64]; } *__dst = dst;

        /*
         * MOVDIR64B %(rdx), rax.
         *
         * Both __src and __dst must be memory constraints in order to tell the
         * compiler that no other memory accesses should be reordered around
         * this one.
         *
         * Also, both must be supplied as lvalues because this tells
         * the compiler what the object is (its size) the instruction accesses.
         * I.e., not the pointers but what they point to, thus the deref'ing '*'.
         */
        asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
                     : "+m" (*__dst)
                     :  "m" (*__src), "a" (__dst), "d" (__src));
}

static inline void movdir64b_io(void __iomem *dst, const void *src)
{
        movdir64b((void __force *)dst, src);
}

/**
 * enqcmds - Enqueue a command in supervisor (CPL0) mode
 * @dst: destination, in MMIO space (must be 512-bit aligned)
 * @src: 512 bits memory operand
 *
 * The ENQCMDS instruction allows software to write a 512-bit command to
 * a 512-bit-aligned special MMIO region that supports the instruction.
 * A return status is loaded into the ZF flag in the RFLAGS register.
 * ZF = 0 equates to success, and ZF = 1 indicates retry or error.
 *
 * This function issues the ENQCMDS instruction to submit data from
 * kernel space to MMIO space, in a unit of 512 bits. Order of data access
 * is not guaranteed, nor is a memory barrier performed afterwards. It
 * returns 0 on success and -EAGAIN on failure.
 *
 * Warning: Do not use this helper unless your driver has checked that the
 * ENQCMDS instruction is supported on the platform and the device accepts
 * ENQCMDS.
 */
static inline int enqcmds(void __iomem *dst, const void *src)
{
        const struct { char _[64]; } *__src = src;
        struct { char _[64]; } __iomem *__dst = dst;
        bool zf;

        /*
         * ENQCMDS %(rdx), rax
         *
         * See movdir64b()'s comment on operand specification.
         */
        asm volatile(".byte 0xf3, 0x0f, 0x38, 0xf8, 0x02, 0x66, 0x90"
                     CC_SET(z)
                     : CC_OUT(z) (zf), "+m" (*__dst)
                     : "m" (*__src), "a" (__dst), "d" (__src));

        /* Submission failure is indicated via EFLAGS.ZF=1 */
        if (zf)
                return -EAGAIN;

        return 0;
}

static __always_inline void tile_release(void)
{
        /*
         * Instruction opcode for TILERELEASE; supported in binutils
         * version >= 2.36.
         */
        asm volatile(".byte 0xc4, 0xe2, 0x78, 0x49, 0xc0");
}

#endif /* __KERNEL__ */

#endif /* _ASM_X86_SPECIAL_INSNS_H */












































































































































































































































































































































































































































































































































































    7 



    7 




    7 























   16 





   17 







   17 
   17 
   17 















   16 




   17 




















































































































    9 






    9 












    9 




    9 




   17 




   17 




























































































































































































































   38 

























































































































































































































































































   26 



   26 

   26 

























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
// SPDX-License-Identifier: GPL-2.0
/*
 * cfg80211 MLME SAP interface
 *
 * Copyright (c) 2009, Jouni Malinen <j@w1.fi>
 * Copyright (c) 2015                Intel Deutschland GmbH
 * Copyright (C) 2019-2020, 2022-2025 Intel Corporation
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/etherdevice.h>
#include <linux/netdevice.h>
#include <linux/nl80211.h>
#include <linux/slab.h>
#include <linux/wireless.h>
#include <net/cfg80211.h>
#include <net/iw_handler.h>
#include "core.h"
#include "nl80211.h"
#include "rdev-ops.h"


void cfg80211_rx_assoc_resp(struct net_device *dev,
                            const struct cfg80211_rx_assoc_resp_data *data)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct wiphy *wiphy = wdev->wiphy;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);
        struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *)data->buf;
        struct cfg80211_connect_resp_params cr = {
                .timeout_reason = NL80211_TIMEOUT_UNSPECIFIED,
                .req_ie = data->req_ies,
                .req_ie_len = data->req_ies_len,
                .resp_ie = mgmt->u.assoc_resp.variable,
                .resp_ie_len = data->len -
                               offsetof(struct ieee80211_mgmt,
                                        u.assoc_resp.variable),
                .status = le16_to_cpu(mgmt->u.assoc_resp.status_code),
                .ap_mld_addr = data->ap_mld_addr,
        };
        unsigned int link_id;

        for (link_id = 0; link_id < ARRAY_SIZE(data->links); link_id++) {
                cr.links[link_id].status = data->links[link_id].status;
                cr.links[link_id].bss = data->links[link_id].bss;

                WARN_ON_ONCE(cr.links[link_id].status != WLAN_STATUS_SUCCESS &&
                             (!cr.ap_mld_addr || !cr.links[link_id].bss));

                if (!cr.links[link_id].bss)
                        continue;
                cr.links[link_id].bssid = data->links[link_id].bss->bssid;
                cr.links[link_id].addr = data->links[link_id].addr;
                /* need to have local link addresses for MLO connections */
                WARN_ON(cr.ap_mld_addr &&
                        !is_valid_ether_addr(cr.links[link_id].addr));

                BUG_ON(!cr.links[link_id].bss->channel);

                if (cr.links[link_id].bss->channel->band == NL80211_BAND_S1GHZ) {
                        WARN_ON(link_id);
                        cr.resp_ie = (u8 *)&mgmt->u.s1g_assoc_resp.variable;
                        cr.resp_ie_len = data->len -
                                         offsetof(struct ieee80211_mgmt,
                                                  u.s1g_assoc_resp.variable);
                }

                if (cr.ap_mld_addr)
                        cr.valid_links |= BIT(link_id);
        }

        trace_cfg80211_send_rx_assoc(dev, data);

        /*
         * This is a bit of a hack, we don't notify userspace of
         * a (re-)association reply if we tried to send a reassoc
         * and got a reject -- we only try again with an assoc
         * frame instead of reassoc.
         */
        if (cfg80211_sme_rx_assoc_resp(wdev, cr.status)) {
                for (link_id = 0; link_id < ARRAY_SIZE(data->links); link_id++) {
                        struct cfg80211_bss *bss = data->links[link_id].bss;

                        if (!bss)
                                continue;

                        cfg80211_unhold_bss(bss_from_pub(bss));
                        cfg80211_put_bss(wiphy, bss);
                }
                return;
        }

        nl80211_send_rx_assoc(rdev, dev, data);
        /* update current_bss etc., consumes the bss reference */
        __cfg80211_connect_result(dev, &cr, cr.status == WLAN_STATUS_SUCCESS);
}
EXPORT_SYMBOL(cfg80211_rx_assoc_resp);

static void cfg80211_process_auth(struct wireless_dev *wdev,
                                  const u8 *buf, size_t len)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);

        nl80211_send_rx_auth(rdev, wdev->netdev, buf, len, GFP_KERNEL);
        cfg80211_sme_rx_auth(wdev, buf, len);
}

static void cfg80211_process_deauth(struct wireless_dev *wdev,
                                    const u8 *buf, size_t len,
                                    bool reconnect)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *)buf;
        const u8 *bssid = mgmt->bssid;
        u16 reason_code = le16_to_cpu(mgmt->u.deauth.reason_code);
        bool from_ap = !ether_addr_equal(mgmt->sa, wdev->netdev->dev_addr);

        nl80211_send_deauth(rdev, wdev->netdev, buf, len, reconnect, GFP_KERNEL);

        if (!wdev->connected || !ether_addr_equal(wdev->u.client.connected_addr, bssid))
                return;

        __cfg80211_disconnected(wdev->netdev, NULL, 0, reason_code, from_ap);
        cfg80211_sme_deauth(wdev);
}

static void cfg80211_process_disassoc(struct wireless_dev *wdev,
                                      const u8 *buf, size_t len,
                                      bool reconnect)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *)buf;
        const u8 *bssid = mgmt->bssid;
        u16 reason_code = le16_to_cpu(mgmt->u.disassoc.reason_code);
        bool from_ap = !ether_addr_equal(mgmt->sa, wdev->netdev->dev_addr);

        nl80211_send_disassoc(rdev, wdev->netdev, buf, len, reconnect,
                              GFP_KERNEL);

        if (WARN_ON(!wdev->connected ||
                    !ether_addr_equal(wdev->u.client.connected_addr, bssid)))
                return;

        __cfg80211_disconnected(wdev->netdev, NULL, 0, reason_code, from_ap);
        cfg80211_sme_disassoc(wdev);
}

void cfg80211_rx_mlme_mgmt(struct net_device *dev, const u8 *buf, size_t len)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct ieee80211_mgmt *mgmt = (void *)buf;

        lockdep_assert_wiphy(wdev->wiphy);

        trace_cfg80211_rx_mlme_mgmt(dev, buf, len);

        if (WARN_ON(len < 2))
                return;

        if (ieee80211_is_auth(mgmt->frame_control))
                cfg80211_process_auth(wdev, buf, len);
        else if (ieee80211_is_deauth(mgmt->frame_control))
                cfg80211_process_deauth(wdev, buf, len, false);
        else if (ieee80211_is_disassoc(mgmt->frame_control))
                cfg80211_process_disassoc(wdev, buf, len, false);
}
EXPORT_SYMBOL(cfg80211_rx_mlme_mgmt);

void cfg80211_auth_timeout(struct net_device *dev, const u8 *addr)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct wiphy *wiphy = wdev->wiphy;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);

        trace_cfg80211_send_auth_timeout(dev, addr);

        nl80211_send_auth_timeout(rdev, dev, addr, GFP_KERNEL);
        cfg80211_sme_auth_timeout(wdev);
}
EXPORT_SYMBOL(cfg80211_auth_timeout);

void cfg80211_assoc_failure(struct net_device *dev,
                            struct cfg80211_assoc_failure *data)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct wiphy *wiphy = wdev->wiphy;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);
        const u8 *addr = data->ap_mld_addr ?: data->bss[0]->bssid;
        int i;

        trace_cfg80211_send_assoc_failure(dev, data);

        if (data->timeout) {
                nl80211_send_assoc_timeout(rdev, dev, addr, GFP_KERNEL);
                cfg80211_sme_assoc_timeout(wdev);
        } else {
                cfg80211_sme_abandon_assoc(wdev);
        }

        for (i = 0; i < ARRAY_SIZE(data->bss); i++) {
                struct cfg80211_bss *bss = data->bss[i];

                if (!bss)
                        continue;

                cfg80211_unhold_bss(bss_from_pub(bss));
                cfg80211_put_bss(wiphy, bss);
        }
}
EXPORT_SYMBOL(cfg80211_assoc_failure);

void cfg80211_tx_mlme_mgmt(struct net_device *dev, const u8 *buf, size_t len,
                           bool reconnect)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct ieee80211_mgmt *mgmt = (void *)buf;

        lockdep_assert_wiphy(wdev->wiphy);

        trace_cfg80211_tx_mlme_mgmt(dev, buf, len, reconnect);

        if (WARN_ON(len < 2))
                return;

        if (ieee80211_is_deauth(mgmt->frame_control))
                cfg80211_process_deauth(wdev, buf, len, reconnect);
        else
                cfg80211_process_disassoc(wdev, buf, len, reconnect);
}
EXPORT_SYMBOL(cfg80211_tx_mlme_mgmt);

void cfg80211_michael_mic_failure(struct net_device *dev, const u8 *addr,
                                  enum nl80211_key_type key_type, int key_id,
                                  const u8 *tsc, gfp_t gfp)
{
        struct wiphy *wiphy = dev->ieee80211_ptr->wiphy;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);
#ifdef CONFIG_CFG80211_WEXT
        union iwreq_data wrqu;
        char *buf = kmalloc(128, gfp);

        if (buf) {
                memset(&wrqu, 0, sizeof(wrqu));
                wrqu.data.length =
                        sprintf(buf, "MLME-MICHAELMICFAILURE."
                                "indication(keyid=%d %scast addr=%pM)",
                                key_id, key_type == NL80211_KEYTYPE_GROUP
                                ? "broad" : "uni", addr);
                wireless_send_event(dev, IWEVCUSTOM, &wrqu, buf);
                kfree(buf);
        }
#endif

        trace_cfg80211_michael_mic_failure(dev, addr, key_type, key_id, tsc);
        nl80211_michael_mic_failure(rdev, dev, addr, key_type, key_id, tsc, gfp);
}
EXPORT_SYMBOL(cfg80211_michael_mic_failure);

/* some MLME handling for userspace SME */
int cfg80211_mlme_auth(struct cfg80211_registered_device *rdev,
                       struct net_device *dev,
                       struct cfg80211_auth_request *req)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;

        lockdep_assert_wiphy(wdev->wiphy);

        if (!req->bss)
                return -ENOENT;

        if (req->link_id >= 0 &&
            !(wdev->wiphy->flags & WIPHY_FLAG_SUPPORTS_MLO))
                return -EINVAL;

        if (req->auth_type == NL80211_AUTHTYPE_SHARED_KEY) {
                if (!req->key || !req->key_len ||
                    req->key_idx < 0 || req->key_idx > 3)
                        return -EINVAL;
        }

        if (wdev->connected &&
            ether_addr_equal(req->bss->bssid, wdev->u.client.connected_addr))
                return -EALREADY;

        if (ether_addr_equal(req->bss->bssid, dev->dev_addr) ||
            (req->link_id >= 0 &&
             ether_addr_equal(req->ap_mld_addr, dev->dev_addr)))
                return -EINVAL;

        return rdev_auth(rdev, dev, req);
}

/*  Do a logical ht_capa &= ht_capa_mask.  */
void cfg80211_oper_and_ht_capa(struct ieee80211_ht_cap *ht_capa,
                               const struct ieee80211_ht_cap *ht_capa_mask)
{
        int i;
        u8 *p1, *p2;
        if (!ht_capa_mask) {
                memset(ht_capa, 0, sizeof(*ht_capa));
                return;
        }

        p1 = (u8*)(ht_capa);
        p2 = (u8*)(ht_capa_mask);
        for (i = 0; i < sizeof(*ht_capa); i++)
                p1[i] &= p2[i];
}

/*  Do a logical vht_capa &= vht_capa_mask.  */
void cfg80211_oper_and_vht_capa(struct ieee80211_vht_cap *vht_capa,
                                const struct ieee80211_vht_cap *vht_capa_mask)
{
        int i;
        u8 *p1, *p2;
        if (!vht_capa_mask) {
                memset(vht_capa, 0, sizeof(*vht_capa));
                return;
        }

        p1 = (u8*)(vht_capa);
        p2 = (u8*)(vht_capa_mask);
        for (i = 0; i < sizeof(*vht_capa); i++)
                p1[i] &= p2[i];
}

static int
cfg80211_mlme_check_mlo_compat(const struct ieee80211_multi_link_elem *mle_a,
                               const struct ieee80211_multi_link_elem *mle_b,
                               struct netlink_ext_ack *extack)
{
        const struct ieee80211_mle_basic_common_info *common_a, *common_b;

        common_a = (const void *)mle_a->variable;
        common_b = (const void *)mle_b->variable;

        if (memcmp(common_a->mld_mac_addr, common_b->mld_mac_addr, ETH_ALEN)) {
                NL_SET_ERR_MSG(extack, "AP MLD address mismatch");
                return -EINVAL;
        }

        if (ieee80211_mle_get_eml_cap((const u8 *)mle_a) !=
            ieee80211_mle_get_eml_cap((const u8 *)mle_b)) {
                NL_SET_ERR_MSG(extack, "link EML capabilities mismatch");
                return -EINVAL;
        }

        if (ieee80211_mle_get_mld_capa_op((const u8 *)mle_a) !=
            ieee80211_mle_get_mld_capa_op((const u8 *)mle_b)) {
                NL_SET_ERR_MSG(extack, "link MLD capabilities/ops mismatch");
                return -EINVAL;
        }

        if (ieee80211_mle_get_ext_mld_capa_op((const u8 *)mle_a) !=
            ieee80211_mle_get_ext_mld_capa_op((const u8 *)mle_b)) {
                NL_SET_ERR_MSG(extack,
                               "extended link MLD capabilities/ops mismatch");
                return -EINVAL;
        }

        return 0;
}

static int cfg80211_mlme_check_mlo(struct net_device *dev,
                                   struct cfg80211_assoc_request *req,
                                   struct netlink_ext_ack *extack)
{
        const struct ieee80211_multi_link_elem *mles[ARRAY_SIZE(req->links)] = {};
        int i;

        if (req->link_id < 0)
                return 0;

        if (!req->links[req->link_id].bss) {
                NL_SET_ERR_MSG(extack, "no BSS for assoc link");
                return -EINVAL;
        }

        rcu_read_lock();
        for (i = 0; i < ARRAY_SIZE(req->links); i++) {
                const struct cfg80211_bss_ies *ies;
                const struct element *ml;

                if (!req->links[i].bss)
                        continue;

                if (ether_addr_equal(req->links[i].bss->bssid, dev->dev_addr)) {
                        NL_SET_ERR_MSG(extack, "BSSID must not be our address");
                        req->links[i].error = -EINVAL;
                        goto error;
                }

                ies = rcu_dereference(req->links[i].bss->ies);
                ml = cfg80211_find_ext_elem(WLAN_EID_EXT_EHT_MULTI_LINK,
                                            ies->data, ies->len);
                if (!ml) {
                        NL_SET_ERR_MSG(extack, "MLO BSS w/o ML element");
                        req->links[i].error = -EINVAL;
                        goto error;
                }

                if (!ieee80211_mle_type_ok(ml->data + 1,
                                           IEEE80211_ML_CONTROL_TYPE_BASIC,
                                           ml->datalen - 1)) {
                        NL_SET_ERR_MSG(extack, "BSS with invalid ML element");
                        req->links[i].error = -EINVAL;
                        goto error;
                }

                mles[i] = (const void *)(ml->data + 1);

                if (ieee80211_mle_get_link_id((const u8 *)mles[i]) != i) {
                        NL_SET_ERR_MSG(extack, "link ID mismatch");
                        req->links[i].error = -EINVAL;
                        goto error;
                }
        }

        if (WARN_ON(!mles[req->link_id]))
                goto error;

        for (i = 0; i < ARRAY_SIZE(req->links); i++) {
                if (i == req->link_id || !req->links[i].bss)
                        continue;

                if (WARN_ON(!mles[i]))
                        goto error;

                if (cfg80211_mlme_check_mlo_compat(mles[req->link_id], mles[i],
                                                   extack)) {
                        req->links[i].error = -EINVAL;
                        goto error;
                }
        }

        rcu_read_unlock();
        return 0;
error:
        rcu_read_unlock();
        return -EINVAL;
}

/* Note: caller must cfg80211_put_bss() regardless of result */
int cfg80211_mlme_assoc(struct cfg80211_registered_device *rdev,
                        struct net_device *dev,
                        struct cfg80211_assoc_request *req,
                        struct netlink_ext_ack *extack)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        int err;

        lockdep_assert_wiphy(wdev->wiphy);

        err = cfg80211_mlme_check_mlo(dev, req, extack);
        if (err)
                return err;

        if (wdev->connected &&
            (!req->prev_bssid ||
             !ether_addr_equal(wdev->u.client.connected_addr, req->prev_bssid)))
                return -EALREADY;

        if ((req->bss && ether_addr_equal(req->bss->bssid, dev->dev_addr)) ||
            (req->link_id >= 0 &&
             ether_addr_equal(req->ap_mld_addr, dev->dev_addr)))
                return -EINVAL;

        cfg80211_oper_and_ht_capa(&req->ht_capa_mask,
                                  rdev->wiphy.ht_capa_mod_mask);
        cfg80211_oper_and_vht_capa(&req->vht_capa_mask,
                                   rdev->wiphy.vht_capa_mod_mask);

        err = rdev_assoc(rdev, dev, req);
        if (!err) {
                int link_id;

                if (req->bss) {
                        cfg80211_ref_bss(&rdev->wiphy, req->bss);
                        cfg80211_hold_bss(bss_from_pub(req->bss));
                }

                for (link_id = 0; link_id < ARRAY_SIZE(req->links); link_id++) {
                        if (!req->links[link_id].bss)
                                continue;
                        cfg80211_ref_bss(&rdev->wiphy, req->links[link_id].bss);
                        cfg80211_hold_bss(bss_from_pub(req->links[link_id].bss));
                }
        }
        return err;
}

int cfg80211_mlme_deauth(struct cfg80211_registered_device *rdev,
                         struct net_device *dev, const u8 *bssid,
                         const u8 *ie, int ie_len, u16 reason,
                         bool local_state_change)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_deauth_request req = {
                .bssid = bssid,
                .reason_code = reason,
                .ie = ie,
                .ie_len = ie_len,
                .local_state_change = local_state_change,
        };

        lockdep_assert_wiphy(wdev->wiphy);

        if (local_state_change &&
            (!wdev->connected ||
             !ether_addr_equal(wdev->u.client.connected_addr, bssid)))
                return 0;

        if (ether_addr_equal(wdev->disconnect_bssid, bssid) ||
            (wdev->connected &&
             ether_addr_equal(wdev->u.client.connected_addr, bssid)))
                wdev->conn_owner_nlportid = 0;

        return rdev_deauth(rdev, dev, &req);
}

int cfg80211_mlme_disassoc(struct cfg80211_registered_device *rdev,
                           struct net_device *dev, const u8 *ap_addr,
                           const u8 *ie, int ie_len, u16 reason,
                           bool local_state_change)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_disassoc_request req = {
                .reason_code = reason,
                .local_state_change = local_state_change,
                .ie = ie,
                .ie_len = ie_len,
                .ap_addr = ap_addr,
        };
        int err;

        lockdep_assert_wiphy(wdev->wiphy);

        if (!wdev->connected)
                return -ENOTCONN;

        if (memcmp(wdev->u.client.connected_addr, ap_addr, ETH_ALEN))
                return -ENOTCONN;

        err = rdev_disassoc(rdev, dev, &req);
        if (err)
                return err;

        /* driver should have reported the disassoc */
        WARN_ON(wdev->connected);
        return 0;
}

void cfg80211_mlme_down(struct cfg80211_registered_device *rdev,
                        struct net_device *dev)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        u8 bssid[ETH_ALEN];

        lockdep_assert_wiphy(wdev->wiphy);

        if (!rdev->ops->deauth)
                return;

        if (!wdev->connected)
                return;

        memcpy(bssid, wdev->u.client.connected_addr, ETH_ALEN);
        cfg80211_mlme_deauth(rdev, dev, bssid, NULL, 0,
                             WLAN_REASON_DEAUTH_LEAVING, false);
}

struct cfg80211_mgmt_registration {
        struct list_head list;
        struct wireless_dev *wdev;

        u32 nlportid;

        int match_len;

        __le16 frame_type;

        bool multicast_rx;

        u8 match[];
};

static void cfg80211_mgmt_registrations_update(struct wireless_dev *wdev)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        struct wireless_dev *tmp;
        struct cfg80211_mgmt_registration *reg;
        struct mgmt_frame_regs upd = {};

        lockdep_assert_held(&rdev->wiphy.mtx);

        spin_lock_bh(&rdev->mgmt_registrations_lock);
        if (!wdev->mgmt_registrations_need_update) {
                spin_unlock_bh(&rdev->mgmt_registrations_lock);
                return;
        }

        rcu_read_lock();
        list_for_each_entry_rcu(tmp, &rdev->wiphy.wdev_list, list) {
                list_for_each_entry(reg, &tmp->mgmt_registrations, list) {
                        u32 mask = BIT(le16_to_cpu(reg->frame_type) >> 4);
                        u32 mcast_mask = 0;

                        if (reg->multicast_rx)
                                mcast_mask = mask;

                        upd.global_stypes |= mask;
                        upd.global_mcast_stypes |= mcast_mask;

                        if (tmp == wdev) {
                                upd.interface_stypes |= mask;
                                upd.interface_mcast_stypes |= mcast_mask;
                        }
                }
        }
        rcu_read_unlock();

        wdev->mgmt_registrations_need_update = 0;
        spin_unlock_bh(&rdev->mgmt_registrations_lock);

        rdev_update_mgmt_frame_registrations(rdev, wdev, &upd);
}

void cfg80211_mgmt_registrations_update_wk(struct work_struct *wk)
{
        struct cfg80211_registered_device *rdev;
        struct wireless_dev *wdev;

        rdev = container_of(wk, struct cfg80211_registered_device,
                            mgmt_registrations_update_wk);

        guard(wiphy)(&rdev->wiphy);

        list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list)
                cfg80211_mgmt_registrations_update(wdev);
}

int cfg80211_mlme_register_mgmt(struct wireless_dev *wdev, u32 snd_portid,
                                u16 frame_type, const u8 *match_data,
                                int match_len, bool multicast_rx,
                                struct netlink_ext_ack *extack)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        struct cfg80211_mgmt_registration *reg, *nreg;
        int err = 0;
        u16 mgmt_type;
        bool update_multicast = false;

        if (!wdev->wiphy->mgmt_stypes)
                return -EOPNOTSUPP;

        if ((frame_type & IEEE80211_FCTL_FTYPE) != IEEE80211_FTYPE_MGMT) {
                NL_SET_ERR_MSG(extack, "frame type not management");
                return -EINVAL;
        }

        if (frame_type & ~(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) {
                NL_SET_ERR_MSG(extack, "Invalid frame type");
                return -EINVAL;
        }

        mgmt_type = (frame_type & IEEE80211_FCTL_STYPE) >> 4;
        if (!(wdev->wiphy->mgmt_stypes[wdev->iftype].rx & BIT(mgmt_type))) {
                NL_SET_ERR_MSG(extack,
                               "Registration to specific type not supported");
                return -EINVAL;
        }

        /*
         * To support Pre Association Security Negotiation (PASN), registration
         * for authentication frames should be supported. However, as some
         * versions of the user space daemons wrongly register to all types of
         * authentication frames (which might result in unexpected behavior)
         * allow such registration if the request is for a specific
         * authentication algorithm number.
         */
        if (wdev->iftype == NL80211_IFTYPE_STATION &&
            (frame_type & IEEE80211_FCTL_STYPE) == IEEE80211_STYPE_AUTH &&
            !(match_data && match_len >= 2)) {
                NL_SET_ERR_MSG(extack,
                               "Authentication algorithm number required");
                return -EINVAL;
        }

        nreg = kzalloc(sizeof(*reg) + match_len, GFP_KERNEL);
        if (!nreg)
                return -ENOMEM;

        spin_lock_bh(&rdev->mgmt_registrations_lock);

        list_for_each_entry(reg, &wdev->mgmt_registrations, list) {
                int mlen = min(match_len, reg->match_len);

                if (frame_type != le16_to_cpu(reg->frame_type))
                        continue;

                if (memcmp(reg->match, match_data, mlen) == 0) {
                        if (reg->multicast_rx != multicast_rx) {
                                update_multicast = true;
                                reg->multicast_rx = multicast_rx;
                                break;
                        }
                        NL_SET_ERR_MSG(extack, "Match already configured");
                        err = -EALREADY;
                        break;
                }
        }

        if (err)
                goto out;

        if (update_multicast) {
                kfree(nreg);
        } else {
                memcpy(nreg->match, match_data, match_len);
                nreg->match_len = match_len;
                nreg->nlportid = snd_portid;
                nreg->frame_type = cpu_to_le16(frame_type);
                nreg->wdev = wdev;
                nreg->multicast_rx = multicast_rx;
                list_add(&nreg->list, &wdev->mgmt_registrations);
        }
        wdev->mgmt_registrations_need_update = 1;
        spin_unlock_bh(&rdev->mgmt_registrations_lock);

        cfg80211_mgmt_registrations_update(wdev);

        return 0;

 out:
        kfree(nreg);
        spin_unlock_bh(&rdev->mgmt_registrations_lock);

        return err;
}

void cfg80211_mlme_unregister_socket(struct wireless_dev *wdev, u32 nlportid)
{
        struct wiphy *wiphy = wdev->wiphy;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);
        struct cfg80211_mgmt_registration *reg, *tmp;

        spin_lock_bh(&rdev->mgmt_registrations_lock);

        list_for_each_entry_safe(reg, tmp, &wdev->mgmt_registrations, list) {
                if (reg->nlportid != nlportid)
                        continue;

                list_del(&reg->list);
                kfree(reg);

                wdev->mgmt_registrations_need_update = 1;
                schedule_work(&rdev->mgmt_registrations_update_wk);
        }

        spin_unlock_bh(&rdev->mgmt_registrations_lock);

        if (nlportid && rdev->crit_proto_nlportid == nlportid) {
                rdev->crit_proto_nlportid = 0;
                rdev_crit_proto_stop(rdev, wdev);
        }

        if (nlportid == wdev->ap_unexpected_nlportid)
                wdev->ap_unexpected_nlportid = 0;
}

void cfg80211_mlme_purge_registrations(struct wireless_dev *wdev)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        struct cfg80211_mgmt_registration *reg, *tmp;

        spin_lock_bh(&rdev->mgmt_registrations_lock);
        list_for_each_entry_safe(reg, tmp, &wdev->mgmt_registrations, list) {
                list_del(&reg->list);
                kfree(reg);
        }
        wdev->mgmt_registrations_need_update = 1;
        spin_unlock_bh(&rdev->mgmt_registrations_lock);

        cfg80211_mgmt_registrations_update(wdev);
}

static bool cfg80211_allowed_address(struct wireless_dev *wdev, const u8 *addr)
{
        int i;

        for_each_valid_link(wdev, i) {
                if (ether_addr_equal(addr, wdev->links[i].addr))
                        return true;
        }

        return ether_addr_equal(addr, wdev_address(wdev));
}

static bool cfg80211_allowed_random_address(struct wireless_dev *wdev,
                                            const struct ieee80211_mgmt *mgmt)
{
        if (ieee80211_is_auth(mgmt->frame_control) ||
            ieee80211_is_deauth(mgmt->frame_control)) {
                /* Allow random TA to be used with authentication and
                 * deauthentication frames if the driver has indicated support.
                 */
                if (wiphy_ext_feature_isset(
                            wdev->wiphy,
                            NL80211_EXT_FEATURE_AUTH_AND_DEAUTH_RANDOM_TA))
                        return true;
        } else if (ieee80211_is_action(mgmt->frame_control) &&
                   mgmt->u.action.category == WLAN_CATEGORY_PUBLIC) {
                /* Allow random TA to be used with Public Action frames if the
                 * driver has indicated support.
                 */
                if (!wdev->connected &&
                    wiphy_ext_feature_isset(
                            wdev->wiphy,
                            NL80211_EXT_FEATURE_MGMT_TX_RANDOM_TA))
                        return true;

                if (wdev->connected &&
                    wiphy_ext_feature_isset(
                            wdev->wiphy,
                            NL80211_EXT_FEATURE_MGMT_TX_RANDOM_TA_CONNECTED))
                        return true;
        }

        return false;
}

int cfg80211_mlme_mgmt_tx(struct cfg80211_registered_device *rdev,
                          struct wireless_dev *wdev,
                          struct cfg80211_mgmt_tx_params *params, u64 *cookie)
{
        const struct ieee80211_mgmt *mgmt;
        u16 stype;

        lockdep_assert_wiphy(&rdev->wiphy);

        if (!wdev->wiphy->mgmt_stypes)
                return -EOPNOTSUPP;

        if (!rdev->ops->mgmt_tx)
                return -EOPNOTSUPP;

        if (params->len < 24 + 1)
                return -EINVAL;

        mgmt = (const struct ieee80211_mgmt *)params->buf;

        if (!ieee80211_is_mgmt(mgmt->frame_control))
                return -EINVAL;

        stype = le16_to_cpu(mgmt->frame_control) & IEEE80211_FCTL_STYPE;
        if (!(wdev->wiphy->mgmt_stypes[wdev->iftype].tx & BIT(stype >> 4)))
                return -EINVAL;

        if (ieee80211_is_action(mgmt->frame_control) &&
            mgmt->u.action.category != WLAN_CATEGORY_PUBLIC) {
                int err = 0;

                switch (wdev->iftype) {
                case NL80211_IFTYPE_ADHOC:
                        /*
                         * check for IBSS DA must be done by driver as
                         * cfg80211 doesn't track the stations
                         */
                        if (!wdev->u.ibss.current_bss ||
                            !ether_addr_equal(wdev->u.ibss.current_bss->pub.bssid,
                                              mgmt->bssid)) {
                                err = -ENOTCONN;
                                break;
                        }
                        break;
                case NL80211_IFTYPE_STATION:
                case NL80211_IFTYPE_P2P_CLIENT:
                        if (!wdev->connected) {
                                err = -ENOTCONN;
                                break;
                        }

                        /* FIXME: MLD may address this differently */

                        if (!ether_addr_equal(wdev->u.client.connected_addr,
                                              mgmt->bssid)) {
                                err = -ENOTCONN;
                                break;
                        }

                        /* for station, check that DA is the AP */
                        if (!ether_addr_equal(wdev->u.client.connected_addr,
                                              mgmt->da)) {
                                err = -ENOTCONN;
                                break;
                        }
                        break;
                case NL80211_IFTYPE_AP:
                case NL80211_IFTYPE_P2P_GO:
                case NL80211_IFTYPE_AP_VLAN:
                        if (!ether_addr_equal(mgmt->bssid, wdev_address(wdev)) &&
                            (params->link_id < 0 ||
                             !ether_addr_equal(mgmt->bssid,
                                               wdev->links[params->link_id].addr)))
                                err = -EINVAL;
                        break;
                case NL80211_IFTYPE_MESH_POINT:
                        if (!ether_addr_equal(mgmt->sa, mgmt->bssid)) {
                                err = -EINVAL;
                                break;
                        }
                        /*
                         * check for mesh DA must be done by driver as
                         * cfg80211 doesn't track the stations
                         */
                        break;
                case NL80211_IFTYPE_P2P_DEVICE:
                        /*
                         * fall through, P2P device only supports
                         * public action frames
                         */
                case NL80211_IFTYPE_NAN:
                default:
                        err = -EOPNOTSUPP;
                        break;
                }

                if (err)
                        return err;
        }

        if (!cfg80211_allowed_address(wdev, mgmt->sa) &&
            !cfg80211_allowed_random_address(wdev, mgmt))
                return -EINVAL;

        /* Transmit the management frame as requested by user space */
        return rdev_mgmt_tx(rdev, wdev, params, cookie);
}

bool cfg80211_rx_mgmt_ext(struct wireless_dev *wdev,
                          struct cfg80211_rx_info *info)
{
        struct wiphy *wiphy = wdev->wiphy;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);
        struct cfg80211_mgmt_registration *reg;
        const struct ieee80211_txrx_stypes *stypes =
                &wiphy->mgmt_stypes[wdev->iftype];
        struct ieee80211_mgmt *mgmt = (void *)info->buf;
        const u8 *data;
        int data_len;
        bool result = false;
        __le16 ftype = mgmt->frame_control &
                cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE);
        u16 stype;

        trace_cfg80211_rx_mgmt(wdev, info);
        stype = (le16_to_cpu(mgmt->frame_control) & IEEE80211_FCTL_STYPE) >> 4;

        if (!(stypes->rx & BIT(stype))) {
                trace_cfg80211_return_bool(false);
                return false;
        }

        data = info->buf + ieee80211_hdrlen(mgmt->frame_control);
        data_len = info->len - ieee80211_hdrlen(mgmt->frame_control);

        spin_lock_bh(&rdev->mgmt_registrations_lock);

        list_for_each_entry(reg, &wdev->mgmt_registrations, list) {
                if (reg->frame_type != ftype)
                        continue;

                if (reg->match_len > data_len)
                        continue;

                if (memcmp(reg->match, data, reg->match_len))
                        continue;

                /* found match! */

                /* Indicate the received Action frame to user space */
                if (nl80211_send_mgmt(rdev, wdev, reg->nlportid, info,
                                      GFP_ATOMIC))
                        continue;

                result = true;
                break;
        }

        spin_unlock_bh(&rdev->mgmt_registrations_lock);

        trace_cfg80211_return_bool(result);
        return result;
}
EXPORT_SYMBOL(cfg80211_rx_mgmt_ext);

void cfg80211_sched_dfs_chan_update(struct cfg80211_registered_device *rdev)
{
        cancel_delayed_work(&rdev->dfs_update_channels_wk);
        queue_delayed_work(cfg80211_wq, &rdev->dfs_update_channels_wk, 0);
}

void cfg80211_dfs_channels_update_work(struct work_struct *work)
{
        struct delayed_work *delayed_work = to_delayed_work(work);
        struct cfg80211_registered_device *rdev;
        struct cfg80211_chan_def chandef;
        struct ieee80211_supported_band *sband;
        struct ieee80211_channel *c;
        struct wiphy *wiphy;
        bool check_again = false;
        unsigned long timeout, next_time = 0;
        unsigned long time_dfs_update;
        enum nl80211_radar_event radar_event;
        int bandid, i;

        rdev = container_of(delayed_work, struct cfg80211_registered_device,
                            dfs_update_channels_wk);
        wiphy = &rdev->wiphy;

        rtnl_lock();
        for (bandid = 0; bandid < NUM_NL80211_BANDS; bandid++) {
                sband = wiphy->bands[bandid];
                if (!sband)
                        continue;

                for (i = 0; i < sband->n_channels; i++) {
                        c = &sband->channels[i];

                        if (!(c->flags & IEEE80211_CHAN_RADAR))
                                continue;

                        if (c->dfs_state != NL80211_DFS_UNAVAILABLE &&
                            c->dfs_state != NL80211_DFS_AVAILABLE)
                                continue;

                        if (c->dfs_state == NL80211_DFS_UNAVAILABLE) {
                                time_dfs_update = IEEE80211_DFS_MIN_NOP_TIME_MS;
                                radar_event = NL80211_RADAR_NOP_FINISHED;
                        } else {
                                if (regulatory_pre_cac_allowed(wiphy) ||
                                    cfg80211_any_wiphy_oper_chan(wiphy, c))
                                        continue;

                                time_dfs_update = REG_PRE_CAC_EXPIRY_GRACE_MS;
                                radar_event = NL80211_RADAR_PRE_CAC_EXPIRED;
                        }

                        timeout = c->dfs_state_entered +
                                  msecs_to_jiffies(time_dfs_update);

                        if (time_after_eq(jiffies, timeout)) {
                                c->dfs_state = NL80211_DFS_USABLE;
                                c->dfs_state_entered = jiffies;

                                cfg80211_chandef_create(&chandef, c,
                                                        NL80211_CHAN_NO_HT);

                                nl80211_radar_notify(rdev, &chandef,
                                                     radar_event, NULL,
                                                     GFP_ATOMIC);

                                regulatory_propagate_dfs_state(wiphy, &chandef,
                                                               c->dfs_state,
                                                               radar_event);
                                continue;
                        }

                        if (!check_again)
                                next_time = timeout - jiffies;
                        else
                                next_time = min(next_time, timeout - jiffies);
                        check_again = true;
                }
        }
        rtnl_unlock();

        /* reschedule if there are other channels waiting to be cleared again */
        if (check_again)
                queue_delayed_work(cfg80211_wq, &rdev->dfs_update_channels_wk,
                                   next_time);
}


void __cfg80211_radar_event(struct wiphy *wiphy,
                            struct cfg80211_chan_def *chandef,
                            bool offchan, gfp_t gfp)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);

        trace_cfg80211_radar_event(wiphy, chandef, offchan);

        /* only set the chandef supplied channel to unavailable, in
         * case the radar is detected on only one of multiple channels
         * spanned by the chandef.
         */
        cfg80211_set_dfs_state(wiphy, chandef, NL80211_DFS_UNAVAILABLE);

        if (offchan)
                queue_work(cfg80211_wq, &rdev->background_cac_abort_wk);

        cfg80211_sched_dfs_chan_update(rdev);

        nl80211_radar_notify(rdev, chandef, NL80211_RADAR_DETECTED, NULL, gfp);

        memcpy(&rdev->radar_chandef, chandef, sizeof(struct cfg80211_chan_def));
        queue_work(cfg80211_wq, &rdev->propagate_radar_detect_wk);
}
EXPORT_SYMBOL(__cfg80211_radar_event);

void cfg80211_cac_event(struct net_device *netdev,
                        const struct cfg80211_chan_def *chandef,
                        enum nl80211_radar_event event, gfp_t gfp,
                        unsigned int link_id)
{
        struct wireless_dev *wdev = netdev->ieee80211_ptr;
        struct wiphy *wiphy = wdev->wiphy;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);
        unsigned long timeout;

        if (WARN_ON(wdev->valid_links &&
                    !(wdev->valid_links & BIT(link_id))))
                return;

        trace_cfg80211_cac_event(netdev, event, link_id);

        if (WARN_ON(!wdev->links[link_id].cac_started &&
                    event != NL80211_RADAR_CAC_STARTED))
                return;

        switch (event) {
        case NL80211_RADAR_CAC_FINISHED:
                timeout = wdev->links[link_id].cac_start_time +
                          msecs_to_jiffies(wdev->links[link_id].cac_time_ms);
                WARN_ON(!time_after_eq(jiffies, timeout));
                cfg80211_set_dfs_state(wiphy, chandef, NL80211_DFS_AVAILABLE);
                memcpy(&rdev->cac_done_chandef, chandef,
                       sizeof(struct cfg80211_chan_def));
                queue_work(cfg80211_wq, &rdev->propagate_cac_done_wk);
                cfg80211_sched_dfs_chan_update(rdev);
                fallthrough;
        case NL80211_RADAR_CAC_ABORTED:
                wdev->links[link_id].cac_started = false;
                break;
        case NL80211_RADAR_CAC_STARTED:
                wdev->links[link_id].cac_started = true;
                break;
        default:
                WARN_ON(1);
                return;
        }

        nl80211_radar_notify(rdev, chandef, event, netdev, gfp);
}
EXPORT_SYMBOL(cfg80211_cac_event);

static void
__cfg80211_background_cac_event(struct cfg80211_registered_device *rdev,
                                struct wireless_dev *wdev,
                                const struct cfg80211_chan_def *chandef,
                                enum nl80211_radar_event event)
{
        struct wiphy *wiphy = &rdev->wiphy;
        struct net_device *netdev;

        lockdep_assert_wiphy(&rdev->wiphy);

        if (!cfg80211_chandef_valid(chandef))
                return;

        if (!rdev->background_radar_wdev)
                return;

        switch (event) {
        case NL80211_RADAR_CAC_FINISHED:
                cfg80211_set_dfs_state(wiphy, chandef, NL80211_DFS_AVAILABLE);
                memcpy(&rdev->cac_done_chandef, chandef, sizeof(*chandef));
                queue_work(cfg80211_wq, &rdev->propagate_cac_done_wk);
                cfg80211_sched_dfs_chan_update(rdev);
                wdev = rdev->background_radar_wdev;
                break;
        case NL80211_RADAR_CAC_ABORTED:
                if (!cancel_delayed_work(&rdev->background_cac_done_wk))
                        return;
                wdev = rdev->background_radar_wdev;
                break;
        case NL80211_RADAR_CAC_STARTED:
                break;
        default:
                return;
        }

        netdev = wdev ? wdev->netdev : NULL;
        nl80211_radar_notify(rdev, chandef, event, netdev, GFP_KERNEL);
}

static void
cfg80211_background_cac_event(struct cfg80211_registered_device *rdev,
                              const struct cfg80211_chan_def *chandef,
                              enum nl80211_radar_event event)
{
        guard(wiphy)(&rdev->wiphy);

        __cfg80211_background_cac_event(rdev, rdev->background_radar_wdev,
                                        chandef, event);
}

void cfg80211_background_cac_done_wk(struct work_struct *work)
{
        struct delayed_work *delayed_work = to_delayed_work(work);
        struct cfg80211_registered_device *rdev;

        rdev = container_of(delayed_work, struct cfg80211_registered_device,
                            background_cac_done_wk);
        cfg80211_background_cac_event(rdev, &rdev->background_radar_chandef,
                                      NL80211_RADAR_CAC_FINISHED);
}

void cfg80211_background_cac_abort_wk(struct work_struct *work)
{
        struct cfg80211_registered_device *rdev;

        rdev = container_of(work, struct cfg80211_registered_device,
                            background_cac_abort_wk);
        cfg80211_background_cac_event(rdev, &rdev->background_radar_chandef,
                                      NL80211_RADAR_CAC_ABORTED);
}

void cfg80211_background_cac_abort(struct wiphy *wiphy)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);

        queue_work(cfg80211_wq, &rdev->background_cac_abort_wk);
}
EXPORT_SYMBOL(cfg80211_background_cac_abort);

int
cfg80211_start_background_radar_detection(struct cfg80211_registered_device *rdev,
                                          struct wireless_dev *wdev,
                                          struct cfg80211_chan_def *chandef)
{
        unsigned int cac_time_ms;
        int err;

        lockdep_assert_wiphy(&rdev->wiphy);

        if (!wiphy_ext_feature_isset(&rdev->wiphy,
                                     NL80211_EXT_FEATURE_RADAR_BACKGROUND))
                return -EOPNOTSUPP;

        /* Offchannel chain already locked by another wdev */
        if (rdev->background_radar_wdev && rdev->background_radar_wdev != wdev)
                return -EBUSY;

        /* CAC already in progress on the offchannel chain */
        if (rdev->background_radar_wdev == wdev &&
            delayed_work_pending(&rdev->background_cac_done_wk))
                return -EBUSY;

        err = rdev_set_radar_background(rdev, chandef);
        if (err)
                return err;

        cac_time_ms = cfg80211_chandef_dfs_cac_time(&rdev->wiphy, chandef);
        if (!cac_time_ms)
                cac_time_ms = IEEE80211_DFS_MIN_CAC_TIME_MS;

        rdev->background_radar_chandef = *chandef;
        rdev->background_radar_wdev = wdev; /* Get offchain ownership */

        __cfg80211_background_cac_event(rdev, wdev, chandef,
                                        NL80211_RADAR_CAC_STARTED);
        queue_delayed_work(cfg80211_wq, &rdev->background_cac_done_wk,
                           msecs_to_jiffies(cac_time_ms));

        return 0;
}

void cfg80211_stop_background_radar_detection(struct wireless_dev *wdev)
{
        struct wiphy *wiphy = wdev->wiphy;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);

        lockdep_assert_wiphy(wiphy);

        if (wdev != rdev->background_radar_wdev)
                return;

        rdev_set_radar_background(rdev, NULL);
        rdev->background_radar_wdev = NULL; /* Release offchain ownership */

        __cfg80211_background_cac_event(rdev, wdev,
                                        &rdev->background_radar_chandef,
                                        NL80211_RADAR_CAC_ABORTED);
}

int cfg80211_assoc_ml_reconf(struct cfg80211_registered_device *rdev,
                             struct net_device *dev,
                             struct cfg80211_ml_reconf_req *req)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        int err;

        lockdep_assert_wiphy(wdev->wiphy);

        err = rdev_assoc_ml_reconf(rdev, dev, req);
        if (!err) {
                int link_id;

                for (link_id = 0; link_id < IEEE80211_MLD_MAX_NUM_LINKS;
                     link_id++) {
                        if (!req->add_links[link_id].bss)
                                continue;

                        cfg80211_ref_bss(&rdev->wiphy, req->add_links[link_id].bss);
                        cfg80211_hold_bss(bss_from_pub(req->add_links[link_id].bss));
                }
        }

        return err;
}

void cfg80211_mlo_reconf_add_done(struct net_device *dev,
                                  struct cfg80211_mlo_reconf_done_data *data)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct wiphy *wiphy = wdev->wiphy;
        int link_id;

        lockdep_assert_wiphy(wiphy);

        trace_cfg80211_mlo_reconf_add_done(dev, data->added_links,
                                           data->buf, data->len);

        if (WARN_ON(!wdev->valid_links))
                return;

        if (WARN_ON(wdev->iftype != NL80211_IFTYPE_STATION &&
                    wdev->iftype != NL80211_IFTYPE_P2P_CLIENT))
                return;

        /* validate that a BSS is given for each added link */
        for (link_id = 0; link_id < ARRAY_SIZE(data->links); link_id++) {
                struct cfg80211_bss *bss = data->links[link_id].bss;

                if (!(data->added_links & BIT(link_id)))
                        continue;

                if (WARN_ON(!bss))
                        return;
        }

        for (link_id = 0; link_id < ARRAY_SIZE(data->links); link_id++) {
                struct cfg80211_bss *bss = data->links[link_id].bss;

                if (!bss)
                        continue;

                if (data->added_links & BIT(link_id)) {
                        wdev->links[link_id].client.current_bss =
                                bss_from_pub(bss);

                        memcpy(wdev->links[link_id].addr,
                               data->links[link_id].addr,
                               ETH_ALEN);
                } else {
                        cfg80211_unhold_bss(bss_from_pub(bss));
                        cfg80211_put_bss(wiphy, bss);
                }
        }

        wdev->valid_links |= data->added_links;
        nl80211_mlo_reconf_add_done(dev, data);
}
EXPORT_SYMBOL(cfg80211_mlo_reconf_add_done);
























































































 4251 










 4248 







 4249 



 4251 



 4253 


 4249 












 4251 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
// SPDX-License-Identifier: GPL-2.0
/*
 * SHA1 routine optimized to do word accesses rather than byte accesses,
 * and to avoid unnecessary copies into the context array.
 *
 * This was based on the git SHA1 implementation.
 */

#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/module.h>
#include <linux/bitops.h>
#include <linux/string.h>
#include <crypto/sha1.h>
#include <linux/unaligned.h>

/*
 * If you have 32 registers or more, the compiler can (and should)
 * try to change the array[] accesses into registers. However, on
 * machines with less than ~25 registers, that won't really work,
 * and at least gcc will make an unholy mess of it.
 *
 * So to avoid that mess which just slows things down, we force
 * the stores to memory to actually happen (we might be better off
 * with a 'W(t)=(val);asm("":"+m" (W(t))' there instead, as
 * suggested by Artur Skawina - that will also make gcc unable to
 * try to do the silly "optimize away loads" part because it won't
 * see what the value will be).
 *
 * Ben Herrenschmidt reports that on PPC, the C version comes close
 * to the optimized asm with this (ie on PPC you don't want that
 * 'volatile', since there are lots of registers).
 *
 * On ARM we get the best code generation by forcing a full memory barrier
 * between each SHA_ROUND, otherwise gcc happily get wild with spilling and
 * the stack frame size simply explode and performance goes down the drain.
 */

#ifdef CONFIG_X86
  #define setW(x, val) (*(volatile __u32 *)&W(x) = (val))
#elif defined(CONFIG_ARM)
  #define setW(x, val) do { W(x) = (val); __asm__("":::"memory"); } while (0)
#else
  #define setW(x, val) (W(x) = (val))
#endif

/* This "rolls" over the 512-bit array */
#define W(x) (array[(x)&15])

/*
 * Where do we get the source from? The first 16 iterations get it from
 * the input data, the next mix it from the 512-bit array.
 */
#define SHA_SRC(t) get_unaligned_be32((__u32 *)data + t)
#define SHA_MIX(t) rol32(W(t+13) ^ W(t+8) ^ W(t+2) ^ W(t), 1)

#define SHA_ROUND(t, input, fn, constant, A, B, C, D, E) do { \
        __u32 TEMP = input(t); setW(t, TEMP); \
        E += TEMP + rol32(A,5) + (fn) + (constant); \
        B = ror32(B, 2); \
        TEMP = E; E = D; D = C; C = B; B = A; A = TEMP; } while (0)

#define T_0_15(t, A, B, C, D, E)  SHA_ROUND(t, SHA_SRC, (((C^D)&B)^D) , 0x5a827999, A, B, C, D, E )
#define T_16_19(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, (((C^D)&B)^D) , 0x5a827999, A, B, C, D, E )
#define T_20_39(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, (B^C^D) , 0x6ed9eba1, A, B, C, D, E )
#define T_40_59(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, ((B&C)+(D&(B^C))) , 0x8f1bbcdc, A, B, C, D, E )
#define T_60_79(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, (B^C^D) ,  0xca62c1d6, A, B, C, D, E )

/**
 * sha1_transform - single block SHA1 transform (deprecated)
 *
 * @digest: 160 bit digest to update
 * @data:   512 bits of data to hash
 * @array:  16 words of workspace (see note)
 *
 * This function executes SHA-1's internal compression function.  It updates the
 * 160-bit internal state (@digest) with a single 512-bit data block (@data).
 *
 * Don't use this function.  SHA-1 is no longer considered secure.  And even if
 * you do have to use SHA-1, this isn't the correct way to hash something with
 * SHA-1 as this doesn't handle padding and finalization.
 *
 * Note: If the hash is security sensitive, the caller should be sure
 * to clear the workspace. This is left to the caller to avoid
 * unnecessary clears between chained hashing operations.
 */
void sha1_transform(__u32 *digest, const char *data, __u32 *array)
{
        __u32 A, B, C, D, E;
        unsigned int i = 0;

        A = digest[0];
        B = digest[1];
        C = digest[2];
        D = digest[3];
        E = digest[4];

        /* Round 1 - iterations 0-16 take their input from 'data' */
        for (; i < 16; ++i)
                T_0_15(i, A, B, C, D, E);

        /* Round 1 - tail. Input from 512-bit mixing array */
        for (; i < 20; ++i)
                T_16_19(i, A, B, C, D, E);

        /* Round 2 */
        for (; i < 40; ++i)
                T_20_39(i, A, B, C, D, E);

        /* Round 3 */
        for (; i < 60; ++i)
                T_40_59(i, A, B, C, D, E);

        /* Round 4 */
        for (; i < 80; ++i)
                T_60_79(i, A, B, C, D, E);

        digest[0] += A;
        digest[1] += B;
        digest[2] += C;
        digest[3] += D;
        digest[4] += E;
}
EXPORT_SYMBOL(sha1_transform);

/**
 * sha1_init - initialize the vectors for a SHA1 digest
 * @buf: vector to initialize
 */
void sha1_init(__u32 *buf)
{
        buf[0] = 0x67452301;
        buf[1] = 0xefcdab89;
        buf[2] = 0x98badcfe;
        buf[3] = 0x10325476;
        buf[4] = 0xc3d2e1f0;
}
EXPORT_SYMBOL(sha1_init);

MODULE_DESCRIPTION("SHA-1 Algorithm");
MODULE_LICENSE("GPL");


















   19 





   19 

















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
// SPDX-License-Identifier: GPL-2.0
#include <linux/bitops.h>
#include <linux/bug.h>
#include <linux/export.h>
#include <linux/limits.h>
#include <linux/math.h>
#include <linux/minmax.h>
#include <linux/types.h>

#include <linux/reciprocal_div.h>

/*
 * For a description of the algorithm please have a look at
 * include/linux/reciprocal_div.h
 */

struct reciprocal_value reciprocal_value(u32 d)
{
        struct reciprocal_value R;
        u64 m;
        int l;

        l = fls(d - 1);
        m = ((1ULL << 32) * ((1ULL << l) - d));
        do_div(m, d);
        ++m;
        R.m = (u32)m;
        R.sh1 = min(l, 1);
        R.sh2 = max(l - 1, 0);

        return R;
}
EXPORT_SYMBOL(reciprocal_value);

struct reciprocal_value_adv reciprocal_value_adv(u32 d, u8 prec)
{
        struct reciprocal_value_adv R;
        u32 l, post_shift;
        u64 mhigh, mlow;

        /* ceil(log2(d)) */
        l = fls(d - 1);
        /* NOTE: mlow/mhigh could overflow u64 when l == 32. This case needs to
         * be handled before calling "reciprocal_value_adv", please see the
         * comment at include/linux/reciprocal_div.h.
         */
        WARN(l == 32,
             "ceil(log2(0x%08x)) == 32, %s doesn't support such divisor",
             d, __func__);
        post_shift = l;
        mlow = 1ULL << (32 + l);
        do_div(mlow, d);
        mhigh = (1ULL << (32 + l)) + (1ULL << (32 + l - prec));
        do_div(mhigh, d);

        for (; post_shift > 0; post_shift--) {
                u64 lo = mlow >> 1, hi = mhigh >> 1;

                if (lo >= hi)
                        break;

                mlow = lo;
                mhigh = hi;
        }

        R.m = (u32)mhigh;
        R.sh = post_shift;
        R.exp = l;
        R.is_wide_m = mhigh > U32_MAX;

        return R;
}
EXPORT_SYMBOL(reciprocal_value_adv);


























































    1 




    1 









































    5 


















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
// SPDX-License-Identifier: GPL-2.0-only
/*
 * debugfs code for HSR & PRP
 * Copyright (C) 2019 Texas Instruments Incorporated
 *
 * Author(s):
 *        Murali Karicheri <m-karicheri2@ti.com>
 */
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/debugfs.h>
#include "hsr_main.h"
#include "hsr_framereg.h"

static struct dentry *hsr_debugfs_root_dir;

/* hsr_node_table_show - Formats and prints node_table entries */
static int
hsr_node_table_show(struct seq_file *sfp, void *data)
{
        struct hsr_priv *priv = (struct hsr_priv *)sfp->private;
        struct hsr_node *node;

        seq_printf(sfp, "Node Table entries for (%s) device\n",
                   (priv->prot_version == PRP_V1 ? "PRP" : "HSR"));
        seq_puts(sfp, "MAC-Address-A,    MAC-Address-B,    time_in[A], ");
        seq_puts(sfp, "time_in[B], Address-B port, ");
        if (priv->prot_version == PRP_V1)
                seq_puts(sfp, "SAN-A, SAN-B, DAN-P\n");
        else
                seq_puts(sfp, "DAN-H\n");

        rcu_read_lock();
        list_for_each_entry_rcu(node, &priv->node_db, mac_list) {
                /* skip self node */
                if (hsr_addr_is_self(priv, node->macaddress_A))
                        continue;
                seq_printf(sfp, "%pM ", &node->macaddress_A[0]);
                seq_printf(sfp, "%pM ", &node->macaddress_B[0]);
                seq_printf(sfp, "%10lx, ", node->time_in[HSR_PT_SLAVE_A]);
                seq_printf(sfp, "%10lx, ", node->time_in[HSR_PT_SLAVE_B]);
                seq_printf(sfp, "%14x, ", node->addr_B_port);

                if (priv->prot_version == PRP_V1)
                        seq_printf(sfp, "%5x, %5x, %5x\n",
                                   node->san_a, node->san_b,
                                   (node->san_a == 0 && node->san_b == 0));
                else
                        seq_printf(sfp, "%5x\n", 1);
        }
        rcu_read_unlock();
        return 0;
}

DEFINE_SHOW_ATTRIBUTE(hsr_node_table);

void hsr_debugfs_rename(struct net_device *dev)
{
        struct hsr_priv *priv = netdev_priv(dev);
        int err;

        err = debugfs_change_name(priv->node_tbl_root, "%s", dev->name);
        if (err)
                netdev_warn(dev, "failed to rename\n");
}

/* hsr_debugfs_init - create hsr node_table file for dumping
 * the node table
 *
 * Description:
 * When debugfs is configured this routine sets up the node_table file per
 * hsr device for dumping the node_table entries
 */
void hsr_debugfs_init(struct hsr_priv *priv, struct net_device *hsr_dev)
{
        struct dentry *de = NULL;

        de = debugfs_create_dir(hsr_dev->name, hsr_debugfs_root_dir);
        if (IS_ERR(de)) {
                pr_err("Cannot create hsr debugfs directory\n");
                return;
        }

        priv->node_tbl_root = de;

        de = debugfs_create_file("node_table", S_IFREG | 0444,
                                 priv->node_tbl_root, priv,
                                 &hsr_node_table_fops);
        if (IS_ERR(de)) {
                pr_err("Cannot create hsr node_table file\n");
                debugfs_remove(priv->node_tbl_root);
                priv->node_tbl_root = NULL;
                return;
        }
}

/* hsr_debugfs_term - Tear down debugfs intrastructure
 *
 * Description:
 * When Debugfs is configured this routine removes debugfs file system
 * elements that are specific to hsr
 */
void
hsr_debugfs_term(struct hsr_priv *priv)
{
        debugfs_remove_recursive(priv->node_tbl_root);
        priv->node_tbl_root = NULL;
}

void hsr_debugfs_create_root(void)
{
        hsr_debugfs_root_dir = debugfs_create_dir("hsr", NULL);
        if (IS_ERR(hsr_debugfs_root_dir)) {
                pr_err("Cannot create hsr debugfs root directory\n");
                hsr_debugfs_root_dir = NULL;
        }
}

void hsr_debugfs_remove_root(void)
{
        /* debugfs_remove() internally checks NULL and ERROR */
        debugfs_remove(hsr_debugfs_root_dir);
}






























































  157 
  157 


  157 

   24 




    9 
   29 



  157 













































































   98 


  100 


  100 











































































   56 



    7 








   21 

















   56 




























































































   32 

   14 
   37 











  149 
   74 
























    4 

   57 




   56 












  146 








  146 
  146 

  146 











  146 



  146 
   70 









   63 






  146 


  146 

  146 






  146 
  146 












   86 













   86 







   86 



   86 

















   66 




   66 











   66 










   66 































   14 




    5 
   17 


   16 





   44 
    2 
    1 
































   63 





    3 
















   60 


   59 






   18 


   59 

   15 


   56 
   56 



    2 









   46 


    4 

   13 








   56 











   56 































   63 



   63 



















    4 






    1 
    4 



    2 
    4 



    2 




    4 

































    3 



    1 








    4 






   13 







    2 



   11 



    7 













   25 









    2 






    2 
   12 










































































   40 







   38 
    2 
   28 
   11 


    1 









   39 








   14 







   14 


    3 

   11 
   11 
    3 

































    2 



    8 











   42 


   34 

   16 



   32 





    1 



   38 



    8 
    1 



    9 

    3 









    6 
    6 

    2 
    6 



    6 



    4 
    2 

    2 



















   53 






   36 






    6 

   24 


    6 
    1 







   24 
    6 
   29 




   45 






   19 






   27 
    2 

   29 




   12 







    2 










   11 

   11 







   11 



    5 







    7 


























    4 





    3 
    1 

















   43 





    1 




   11 













    1 










   35 


    1 





   70 





   19 

   53 

   53 





























    9 






    9 





    8 


































    8 


    8 



    8 





















  137 


  137 



















































































    7 


    7 
























   10 















    4 

    6 












































   47 


   38 



    6 









    6 





    7 


    1 


    1 


    1 





    2 



    2 






































































































    2 






    2 



    6 








    4 







    5 

    4 











   10 














   10 





   10 






    7 



    7 



















































    9 























































































































































































































































































   61 


   49 



    7 







    1 

    1 

    1 

    2 

    1 



    8 




   18 


    1 


   17 

    3 


    5 



    4 

    5 










    9 





    3 















    2 




















    4 











































   22 


   14 



    6 








    1 

    1 

    3 



    3 





    1 


    1 



    4 





   29 


    2 


    1 



    7 

    1 

    2 




    1 

    5 


    1 







    2 




    2 




    2 




    1 

    2 

    6 























   16 


    1 

    1 



    2 

    1 



    3 

    2 



    1 

    1 







    1 



















    1 











    1 


    1 









   23 


















  342 
   62 



   60 














    4 


   57 














   12 


  313 






   98 
   11 



    8 


   14 


    7 

   92 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Functions to manage eBPF programs attached to cgroups
 *
 * Copyright (c) 2016 Daniel Mack
 */

#include <linux/kernel.h>
#include <linux/atomic.h>
#include <linux/cgroup.h>
#include <linux/filter.h>
#include <linux/slab.h>
#include <linux/sysctl.h>
#include <linux/string.h>
#include <linux/bpf.h>
#include <linux/bpf-cgroup.h>
#include <linux/bpf_lsm.h>
#include <linux/bpf_verifier.h>
#include <net/sock.h>
#include <net/bpf_sk_storage.h>

#include "../cgroup/cgroup-internal.h"

DEFINE_STATIC_KEY_ARRAY_FALSE(cgroup_bpf_enabled_key, MAX_CGROUP_BPF_ATTACH_TYPE);
EXPORT_SYMBOL(cgroup_bpf_enabled_key);

/*
 * cgroup bpf destruction makes heavy use of work items and there can be a lot
 * of concurrent destructions.  Use a separate workqueue so that cgroup bpf
 * destruction work items don't end up filling up max_active of system_wq
 * which may lead to deadlock.
 */
static struct workqueue_struct *cgroup_bpf_destroy_wq;

static int __init cgroup_bpf_wq_init(void)
{
        cgroup_bpf_destroy_wq = alloc_workqueue("cgroup_bpf_destroy", 0, 1);
        if (!cgroup_bpf_destroy_wq)
                panic("Failed to alloc workqueue for cgroup bpf destroy.\n");
        return 0;
}
core_initcall(cgroup_bpf_wq_init);

/* __always_inline is necessary to prevent indirect call through run_prog
 * function pointer.
 */
static __always_inline int
bpf_prog_run_array_cg(const struct cgroup_bpf *cgrp,
                      enum cgroup_bpf_attach_type atype,
                      const void *ctx, bpf_prog_run_fn run_prog,
                      int retval, u32 *ret_flags)
{
        const struct bpf_prog_array_item *item;
        const struct bpf_prog *prog;
        const struct bpf_prog_array *array;
        struct bpf_run_ctx *old_run_ctx;
        struct bpf_cg_run_ctx run_ctx;
        u32 func_ret;

        run_ctx.retval = retval;
        migrate_disable();
        rcu_read_lock();
        array = rcu_dereference(cgrp->effective[atype]);
        item = &array->items[0];
        old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
        while ((prog = READ_ONCE(item->prog))) {
                run_ctx.prog_item = item;
                func_ret = run_prog(prog, ctx);
                if (ret_flags) {
                        *(ret_flags) |= (func_ret >> 1);
                        func_ret &= 1;
                }
                if (!func_ret && !IS_ERR_VALUE((long)run_ctx.retval))
                        run_ctx.retval = -EPERM;
                item++;
        }
        bpf_reset_run_ctx(old_run_ctx);
        rcu_read_unlock();
        migrate_enable();
        return run_ctx.retval;
}

unsigned int __cgroup_bpf_run_lsm_sock(const void *ctx,
                                       const struct bpf_insn *insn)
{
        const struct bpf_prog *shim_prog;
        struct sock *sk;
        struct cgroup *cgrp;
        int ret = 0;
        u64 *args;

        args = (u64 *)ctx;
        sk = (void *)(unsigned long)args[0];
        /*shim_prog = container_of(insn, struct bpf_prog, insnsi);*/
        shim_prog = (const struct bpf_prog *)((void *)insn - offsetof(struct bpf_prog, insnsi));

        cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
        if (likely(cgrp))
                ret = bpf_prog_run_array_cg(&cgrp->bpf,
                                            shim_prog->aux->cgroup_atype,
                                            ctx, bpf_prog_run, 0, NULL);
        return ret;
}

unsigned int __cgroup_bpf_run_lsm_socket(const void *ctx,
                                         const struct bpf_insn *insn)
{
        const struct bpf_prog *shim_prog;
        struct socket *sock;
        struct cgroup *cgrp;
        int ret = 0;
        u64 *args;

        args = (u64 *)ctx;
        sock = (void *)(unsigned long)args[0];
        /*shim_prog = container_of(insn, struct bpf_prog, insnsi);*/
        shim_prog = (const struct bpf_prog *)((void *)insn - offsetof(struct bpf_prog, insnsi));

        cgrp = sock_cgroup_ptr(&sock->sk->sk_cgrp_data);
        if (likely(cgrp))
                ret = bpf_prog_run_array_cg(&cgrp->bpf,
                                            shim_prog->aux->cgroup_atype,
                                            ctx, bpf_prog_run, 0, NULL);
        return ret;
}

unsigned int __cgroup_bpf_run_lsm_current(const void *ctx,
                                          const struct bpf_insn *insn)
{
        const struct bpf_prog *shim_prog;
        struct cgroup *cgrp;
        int ret = 0;

        /*shim_prog = container_of(insn, struct bpf_prog, insnsi);*/
        shim_prog = (const struct bpf_prog *)((void *)insn - offsetof(struct bpf_prog, insnsi));

        /* We rely on trampoline's __bpf_prog_enter_lsm_cgroup to grab RCU read lock. */
        cgrp = task_dfl_cgroup(current);
        if (likely(cgrp))
                ret = bpf_prog_run_array_cg(&cgrp->bpf,
                                            shim_prog->aux->cgroup_atype,
                                            ctx, bpf_prog_run, 0, NULL);
        return ret;
}

#ifdef CONFIG_BPF_LSM
struct cgroup_lsm_atype {
        u32 attach_btf_id;
        int refcnt;
};

static struct cgroup_lsm_atype cgroup_lsm_atype[CGROUP_LSM_NUM];

static enum cgroup_bpf_attach_type
bpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id)
{
        int i;

        lockdep_assert_held(&cgroup_mutex);

        if (attach_type != BPF_LSM_CGROUP)
                return to_cgroup_bpf_attach_type(attach_type);

        for (i = 0; i < ARRAY_SIZE(cgroup_lsm_atype); i++)
                if (cgroup_lsm_atype[i].attach_btf_id == attach_btf_id)
                        return CGROUP_LSM_START + i;

        for (i = 0; i < ARRAY_SIZE(cgroup_lsm_atype); i++)
                if (cgroup_lsm_atype[i].attach_btf_id == 0)
                        return CGROUP_LSM_START + i;

        return -E2BIG;

}

void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype)
{
        int i = cgroup_atype - CGROUP_LSM_START;

        lockdep_assert_held(&cgroup_mutex);

        WARN_ON_ONCE(cgroup_lsm_atype[i].attach_btf_id &&
                     cgroup_lsm_atype[i].attach_btf_id != attach_btf_id);

        cgroup_lsm_atype[i].attach_btf_id = attach_btf_id;
        cgroup_lsm_atype[i].refcnt++;
}

void bpf_cgroup_atype_put(int cgroup_atype)
{
        int i = cgroup_atype - CGROUP_LSM_START;

        cgroup_lock();
        if (--cgroup_lsm_atype[i].refcnt <= 0)
                cgroup_lsm_atype[i].attach_btf_id = 0;
        WARN_ON_ONCE(cgroup_lsm_atype[i].refcnt < 0);
        cgroup_unlock();
}
#else
static enum cgroup_bpf_attach_type
bpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id)
{
        if (attach_type != BPF_LSM_CGROUP)
                return to_cgroup_bpf_attach_type(attach_type);
        return -EOPNOTSUPP;
}
#endif /* CONFIG_BPF_LSM */

void cgroup_bpf_offline(struct cgroup *cgrp)
{
        cgroup_get(cgrp);
        percpu_ref_kill(&cgrp->bpf.refcnt);
}

static void bpf_cgroup_storages_free(struct bpf_cgroup_storage *storages[])
{
        enum bpf_cgroup_storage_type stype;

        for_each_cgroup_storage_type(stype)
                bpf_cgroup_storage_free(storages[stype]);
}

static int bpf_cgroup_storages_alloc(struct bpf_cgroup_storage *storages[],
                                     struct bpf_cgroup_storage *new_storages[],
                                     enum bpf_attach_type type,
                                     struct bpf_prog *prog,
                                     struct cgroup *cgrp)
{
        enum bpf_cgroup_storage_type stype;
        struct bpf_cgroup_storage_key key;
        struct bpf_map *map;

        key.cgroup_inode_id = cgroup_id(cgrp);
        key.attach_type = type;

        for_each_cgroup_storage_type(stype) {
                map = prog->aux->cgroup_storage[stype];
                if (!map)
                        continue;

                storages[stype] = cgroup_storage_lookup((void *)map, &key, false);
                if (storages[stype])
                        continue;

                storages[stype] = bpf_cgroup_storage_alloc(prog, stype);
                if (IS_ERR(storages[stype])) {
                        bpf_cgroup_storages_free(new_storages);
                        return -ENOMEM;
                }

                new_storages[stype] = storages[stype];
        }

        return 0;
}

static void bpf_cgroup_storages_assign(struct bpf_cgroup_storage *dst[],
                                       struct bpf_cgroup_storage *src[])
{
        enum bpf_cgroup_storage_type stype;

        for_each_cgroup_storage_type(stype)
                dst[stype] = src[stype];
}

static void bpf_cgroup_storages_link(struct bpf_cgroup_storage *storages[],
                                     struct cgroup *cgrp,
                                     enum bpf_attach_type attach_type)
{
        enum bpf_cgroup_storage_type stype;

        for_each_cgroup_storage_type(stype)
                bpf_cgroup_storage_link(storages[stype], cgrp, attach_type);
}

/* Called when bpf_cgroup_link is auto-detached from dying cgroup.
 * It drops cgroup and bpf_prog refcounts, and marks bpf_link as defunct. It
 * doesn't free link memory, which will eventually be done by bpf_link's
 * release() callback, when its last FD is closed.
 */
static void bpf_cgroup_link_auto_detach(struct bpf_cgroup_link *link)
{
        cgroup_put(link->cgroup);
        link->cgroup = NULL;
}

/**
 * cgroup_bpf_release() - put references of all bpf programs and
 *                        release all cgroup bpf data
 * @work: work structure embedded into the cgroup to modify
 */
static void cgroup_bpf_release(struct work_struct *work)
{
        struct cgroup *p, *cgrp = container_of(work, struct cgroup,
                                               bpf.release_work);
        struct bpf_prog_array *old_array;
        struct list_head *storages = &cgrp->bpf.storages;
        struct bpf_cgroup_storage *storage, *stmp;

        unsigned int atype;

        cgroup_lock();

        for (atype = 0; atype < ARRAY_SIZE(cgrp->bpf.progs); atype++) {
                struct hlist_head *progs = &cgrp->bpf.progs[atype];
                struct bpf_prog_list *pl;
                struct hlist_node *pltmp;

                hlist_for_each_entry_safe(pl, pltmp, progs, node) {
                        hlist_del(&pl->node);
                        if (pl->prog) {
                                if (pl->prog->expected_attach_type == BPF_LSM_CGROUP)
                                        bpf_trampoline_unlink_cgroup_shim(pl->prog);
                                bpf_prog_put(pl->prog);
                        }
                        if (pl->link) {
                                if (pl->link->link.prog->expected_attach_type == BPF_LSM_CGROUP)
                                        bpf_trampoline_unlink_cgroup_shim(pl->link->link.prog);
                                bpf_cgroup_link_auto_detach(pl->link);
                        }
                        kfree(pl);
                        static_branch_dec(&cgroup_bpf_enabled_key[atype]);
                }
                old_array = rcu_dereference_protected(
                                cgrp->bpf.effective[atype],
                                lockdep_is_held(&cgroup_mutex));
                bpf_prog_array_free(old_array);
        }

        list_for_each_entry_safe(storage, stmp, storages, list_cg) {
                bpf_cgroup_storage_unlink(storage);
                bpf_cgroup_storage_free(storage);
        }

        cgroup_unlock();

        for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))
                cgroup_bpf_put(p);

        percpu_ref_exit(&cgrp->bpf.refcnt);
        cgroup_put(cgrp);
}

/**
 * cgroup_bpf_release_fn() - callback used to schedule releasing
 *                           of bpf cgroup data
 * @ref: percpu ref counter structure
 */
static void cgroup_bpf_release_fn(struct percpu_ref *ref)
{
        struct cgroup *cgrp = container_of(ref, struct cgroup, bpf.refcnt);

        INIT_WORK(&cgrp->bpf.release_work, cgroup_bpf_release);
        queue_work(cgroup_bpf_destroy_wq, &cgrp->bpf.release_work);
}

/* Get underlying bpf_prog of bpf_prog_list entry, regardless if it's through
 * link or direct prog.
 */
static struct bpf_prog *prog_list_prog(struct bpf_prog_list *pl)
{
        if (pl->prog)
                return pl->prog;
        if (pl->link)
                return pl->link->link.prog;
        return NULL;
}

/* count number of elements in the list.
 * it's slow but the list cannot be long
 */
static u32 prog_list_length(struct hlist_head *head, int *preorder_cnt)
{
        struct bpf_prog_list *pl;
        u32 cnt = 0;

        hlist_for_each_entry(pl, head, node) {
                if (!prog_list_prog(pl))
                        continue;
                if (preorder_cnt && (pl->flags & BPF_F_PREORDER))
                        (*preorder_cnt)++;
                cnt++;
        }
        return cnt;
}

/* if parent has non-overridable prog attached,
 * disallow attaching new programs to the descendent cgroup.
 * if parent has overridable or multi-prog, allow attaching
 */
static bool hierarchy_allows_attach(struct cgroup *cgrp,
                                    enum cgroup_bpf_attach_type atype)
{
        struct cgroup *p;

        p = cgroup_parent(cgrp);
        if (!p)
                return true;
        do {
                u32 flags = p->bpf.flags[atype];
                u32 cnt;

                if (flags & BPF_F_ALLOW_MULTI)
                        return true;
                cnt = prog_list_length(&p->bpf.progs[atype], NULL);
                WARN_ON_ONCE(cnt > 1);
                if (cnt == 1)
                        return !!(flags & BPF_F_ALLOW_OVERRIDE);
                p = cgroup_parent(p);
        } while (p);
        return true;
}

/* compute a chain of effective programs for a given cgroup:
 * start from the list of programs in this cgroup and add
 * all parent programs.
 * Note that parent's F_ALLOW_OVERRIDE-type program is yielding
 * to programs in this cgroup
 */
static int compute_effective_progs(struct cgroup *cgrp,
                                   enum cgroup_bpf_attach_type atype,
                                   struct bpf_prog_array **array)
{
        struct bpf_prog_array_item *item;
        struct bpf_prog_array *progs;
        struct bpf_prog_list *pl;
        struct cgroup *p = cgrp;
        int i, j, cnt = 0, preorder_cnt = 0, fstart, bstart, init_bstart;

        /* count number of effective programs by walking parents */
        do {
                if (cnt == 0 || (p->bpf.flags[atype] & BPF_F_ALLOW_MULTI))
                        cnt += prog_list_length(&p->bpf.progs[atype], &preorder_cnt);
                p = cgroup_parent(p);
        } while (p);

        progs = bpf_prog_array_alloc(cnt, GFP_KERNEL);
        if (!progs)
                return -ENOMEM;

        /* populate the array with effective progs */
        cnt = 0;
        p = cgrp;
        fstart = preorder_cnt;
        bstart = preorder_cnt - 1;
        do {
                if (cnt > 0 && !(p->bpf.flags[atype] & BPF_F_ALLOW_MULTI))
                        continue;

                init_bstart = bstart;
                hlist_for_each_entry(pl, &p->bpf.progs[atype], node) {
                        if (!prog_list_prog(pl))
                                continue;

                        if (pl->flags & BPF_F_PREORDER) {
                                item = &progs->items[bstart];
                                bstart--;
                        } else {
                                item = &progs->items[fstart];
                                fstart++;
                        }
                        item->prog = prog_list_prog(pl);
                        bpf_cgroup_storages_assign(item->cgroup_storage,
                                                   pl->storage);
                        cnt++;
                }

                /* reverse pre-ordering progs at this cgroup level */
                for (i = bstart + 1, j = init_bstart; i < j; i++, j--)
                        swap(progs->items[i], progs->items[j]);

        } while ((p = cgroup_parent(p)));

        *array = progs;
        return 0;
}

static void activate_effective_progs(struct cgroup *cgrp,
                                     enum cgroup_bpf_attach_type atype,
                                     struct bpf_prog_array *old_array)
{
        old_array = rcu_replace_pointer(cgrp->bpf.effective[atype], old_array,
                                        lockdep_is_held(&cgroup_mutex));
        /* free prog array after grace period, since __cgroup_bpf_run_*()
         * might be still walking the array
         */
        bpf_prog_array_free(old_array);
}

/**
 * cgroup_bpf_inherit() - inherit effective programs from parent
 * @cgrp: the cgroup to modify
 */
int cgroup_bpf_inherit(struct cgroup *cgrp)
{
/* has to use marco instead of const int, since compiler thinks
 * that array below is variable length
 */
#define        NR ARRAY_SIZE(cgrp->bpf.effective)
        struct bpf_prog_array *arrays[NR] = {};
        struct cgroup *p;
        int ret, i;

        ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0,
                              GFP_KERNEL);
        if (ret)
                return ret;

        for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))
                cgroup_bpf_get(p);

        for (i = 0; i < NR; i++)
                INIT_HLIST_HEAD(&cgrp->bpf.progs[i]);

        INIT_LIST_HEAD(&cgrp->bpf.storages);

        for (i = 0; i < NR; i++)
                if (compute_effective_progs(cgrp, i, &arrays[i]))
                        goto cleanup;

        for (i = 0; i < NR; i++)
                activate_effective_progs(cgrp, i, arrays[i]);

        return 0;
cleanup:
        for (i = 0; i < NR; i++)
                bpf_prog_array_free(arrays[i]);

        for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))
                cgroup_bpf_put(p);

        percpu_ref_exit(&cgrp->bpf.refcnt);

        return -ENOMEM;
}

static int update_effective_progs(struct cgroup *cgrp,
                                  enum cgroup_bpf_attach_type atype)
{
        struct cgroup_subsys_state *css;
        int err;

        /* allocate and recompute effective prog arrays */
        css_for_each_descendant_pre(css, &cgrp->self) {
                struct cgroup *desc = container_of(css, struct cgroup, self);

                if (percpu_ref_is_zero(&desc->bpf.refcnt))
                        continue;

                err = compute_effective_progs(desc, atype, &desc->bpf.inactive);
                if (err)
                        goto cleanup;
        }

        /* all allocations were successful. Activate all prog arrays */
        css_for_each_descendant_pre(css, &cgrp->self) {
                struct cgroup *desc = container_of(css, struct cgroup, self);

                if (percpu_ref_is_zero(&desc->bpf.refcnt)) {
                        if (unlikely(desc->bpf.inactive)) {
                                bpf_prog_array_free(desc->bpf.inactive);
                                desc->bpf.inactive = NULL;
                        }
                        continue;
                }

                activate_effective_progs(desc, atype, desc->bpf.inactive);
                desc->bpf.inactive = NULL;
        }

        return 0;

cleanup:
        /* oom while computing effective. Free all computed effective arrays
         * since they were not activated
         */
        css_for_each_descendant_pre(css, &cgrp->self) {
                struct cgroup *desc = container_of(css, struct cgroup, self);

                bpf_prog_array_free(desc->bpf.inactive);
                desc->bpf.inactive = NULL;
        }

        return err;
}

#define BPF_CGROUP_MAX_PROGS 64

static struct bpf_prog_list *find_attach_entry(struct hlist_head *progs,
                                               struct bpf_prog *prog,
                                               struct bpf_cgroup_link *link,
                                               struct bpf_prog *replace_prog,
                                               bool allow_multi)
{
        struct bpf_prog_list *pl;

        /* single-attach case */
        if (!allow_multi) {
                if (hlist_empty(progs))
                        return NULL;
                return hlist_entry(progs->first, typeof(*pl), node);
        }

        hlist_for_each_entry(pl, progs, node) {
                if (prog && pl->prog == prog && prog != replace_prog)
                        /* disallow attaching the same prog twice */
                        return ERR_PTR(-EINVAL);
                if (link && pl->link == link)
                        /* disallow attaching the same link twice */
                        return ERR_PTR(-EINVAL);
        }

        /* direct prog multi-attach w/ replacement case */
        if (replace_prog) {
                hlist_for_each_entry(pl, progs, node) {
                        if (pl->prog == replace_prog)
                                /* a match found */
                                return pl;
                }
                /* prog to replace not found for cgroup */
                return ERR_PTR(-ENOENT);
        }

        return NULL;
}

/**
 * __cgroup_bpf_attach() - Attach the program or the link to a cgroup, and
 *                         propagate the change to descendants
 * @cgrp: The cgroup which descendants to traverse
 * @prog: A program to attach
 * @link: A link to attach
 * @replace_prog: Previously attached program to replace if BPF_F_REPLACE is set
 * @type: Type of attach operation
 * @flags: Option flags
 *
 * Exactly one of @prog or @link can be non-null.
 * Must be called with cgroup_mutex held.
 */
static int __cgroup_bpf_attach(struct cgroup *cgrp,
                               struct bpf_prog *prog, struct bpf_prog *replace_prog,
                               struct bpf_cgroup_link *link,
                               enum bpf_attach_type type, u32 flags)
{
        u32 saved_flags = (flags & (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI));
        struct bpf_prog *old_prog = NULL;
        struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {};
        struct bpf_cgroup_storage *new_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {};
        struct bpf_prog *new_prog = prog ? : link->link.prog;
        enum cgroup_bpf_attach_type atype;
        struct bpf_prog_list *pl;
        struct hlist_head *progs;
        int err;

        if (((flags & BPF_F_ALLOW_OVERRIDE) && (flags & BPF_F_ALLOW_MULTI)) ||
            ((flags & BPF_F_REPLACE) && !(flags & BPF_F_ALLOW_MULTI)))
                /* invalid combination */
                return -EINVAL;
        if (link && (prog || replace_prog))
                /* only either link or prog/replace_prog can be specified */
                return -EINVAL;
        if (!!replace_prog != !!(flags & BPF_F_REPLACE))
                /* replace_prog implies BPF_F_REPLACE, and vice versa */
                return -EINVAL;

        atype = bpf_cgroup_atype_find(type, new_prog->aux->attach_btf_id);
        if (atype < 0)
                return -EINVAL;

        progs = &cgrp->bpf.progs[atype];

        if (!hierarchy_allows_attach(cgrp, atype))
                return -EPERM;

        if (!hlist_empty(progs) && cgrp->bpf.flags[atype] != saved_flags)
                /* Disallow attaching non-overridable on top
                 * of existing overridable in this cgroup.
                 * Disallow attaching multi-prog if overridable or none
                 */
                return -EPERM;

        if (prog_list_length(progs, NULL) >= BPF_CGROUP_MAX_PROGS)
                return -E2BIG;

        pl = find_attach_entry(progs, prog, link, replace_prog,
                               flags & BPF_F_ALLOW_MULTI);
        if (IS_ERR(pl))
                return PTR_ERR(pl);

        if (bpf_cgroup_storages_alloc(storage, new_storage, type,
                                      prog ? : link->link.prog, cgrp))
                return -ENOMEM;

        if (pl) {
                old_prog = pl->prog;
        } else {
                struct hlist_node *last = NULL;

                pl = kmalloc(sizeof(*pl), GFP_KERNEL);
                if (!pl) {
                        bpf_cgroup_storages_free(new_storage);
                        return -ENOMEM;
                }
                if (hlist_empty(progs))
                        hlist_add_head(&pl->node, progs);
                else
                        hlist_for_each(last, progs) {
                                if (last->next)
                                        continue;
                                hlist_add_behind(&pl->node, last);
                                break;
                        }
        }

        pl->prog = prog;
        pl->link = link;
        pl->flags = flags;
        bpf_cgroup_storages_assign(pl->storage, storage);
        cgrp->bpf.flags[atype] = saved_flags;

        if (type == BPF_LSM_CGROUP) {
                err = bpf_trampoline_link_cgroup_shim(new_prog, atype);
                if (err)
                        goto cleanup;
        }

        err = update_effective_progs(cgrp, atype);
        if (err)
                goto cleanup_trampoline;

        if (old_prog) {
                if (type == BPF_LSM_CGROUP)
                        bpf_trampoline_unlink_cgroup_shim(old_prog);
                bpf_prog_put(old_prog);
        } else {
                static_branch_inc(&cgroup_bpf_enabled_key[atype]);
        }
        bpf_cgroup_storages_link(new_storage, cgrp, type);
        return 0;

cleanup_trampoline:
        if (type == BPF_LSM_CGROUP)
                bpf_trampoline_unlink_cgroup_shim(new_prog);

cleanup:
        if (old_prog) {
                pl->prog = old_prog;
                pl->link = NULL;
        }
        bpf_cgroup_storages_free(new_storage);
        if (!old_prog) {
                hlist_del(&pl->node);
                kfree(pl);
        }
        return err;
}

static int cgroup_bpf_attach(struct cgroup *cgrp,
                             struct bpf_prog *prog, struct bpf_prog *replace_prog,
                             struct bpf_cgroup_link *link,
                             enum bpf_attach_type type,
                             u32 flags)
{
        int ret;

        cgroup_lock();
        ret = __cgroup_bpf_attach(cgrp, prog, replace_prog, link, type, flags);
        cgroup_unlock();
        return ret;
}

/* Swap updated BPF program for given link in effective program arrays across
 * all descendant cgroups. This function is guaranteed to succeed.
 */
static void replace_effective_prog(struct cgroup *cgrp,
                                   enum cgroup_bpf_attach_type atype,
                                   struct bpf_cgroup_link *link)
{
        struct bpf_prog_array_item *item;
        struct cgroup_subsys_state *css;
        struct bpf_prog_array *progs;
        struct bpf_prog_list *pl;
        struct hlist_head *head;
        struct cgroup *cg;
        int pos;

        css_for_each_descendant_pre(css, &cgrp->self) {
                struct cgroup *desc = container_of(css, struct cgroup, self);

                if (percpu_ref_is_zero(&desc->bpf.refcnt))
                        continue;

                /* find position of link in effective progs array */
                for (pos = 0, cg = desc; cg; cg = cgroup_parent(cg)) {
                        if (pos && !(cg->bpf.flags[atype] & BPF_F_ALLOW_MULTI))
                                continue;

                        head = &cg->bpf.progs[atype];
                        hlist_for_each_entry(pl, head, node) {
                                if (!prog_list_prog(pl))
                                        continue;
                                if (pl->link == link)
                                        goto found;
                                pos++;
                        }
                }
found:
                BUG_ON(!cg);
                progs = rcu_dereference_protected(
                                desc->bpf.effective[atype],
                                lockdep_is_held(&cgroup_mutex));
                item = &progs->items[pos];
                WRITE_ONCE(item->prog, link->link.prog);
        }
}

/**
 * __cgroup_bpf_replace() - Replace link's program and propagate the change
 *                          to descendants
 * @cgrp: The cgroup which descendants to traverse
 * @link: A link for which to replace BPF program
 * @new_prog: &struct bpf_prog for the target BPF program with its refcnt
 *            incremented
 *
 * Must be called with cgroup_mutex held.
 */
static int __cgroup_bpf_replace(struct cgroup *cgrp,
                                struct bpf_cgroup_link *link,
                                struct bpf_prog *new_prog)
{
        enum cgroup_bpf_attach_type atype;
        struct bpf_prog *old_prog;
        struct bpf_prog_list *pl;
        struct hlist_head *progs;
        bool found = false;

        atype = bpf_cgroup_atype_find(link->type, new_prog->aux->attach_btf_id);
        if (atype < 0)
                return -EINVAL;

        progs = &cgrp->bpf.progs[atype];

        if (link->link.prog->type != new_prog->type)
                return -EINVAL;

        hlist_for_each_entry(pl, progs, node) {
                if (pl->link == link) {
                        found = true;
                        break;
                }
        }
        if (!found)
                return -ENOENT;

        old_prog = xchg(&link->link.prog, new_prog);
        replace_effective_prog(cgrp, atype, link);
        bpf_prog_put(old_prog);
        return 0;
}

static int cgroup_bpf_replace(struct bpf_link *link, struct bpf_prog *new_prog,
                              struct bpf_prog *old_prog)
{
        struct bpf_cgroup_link *cg_link;
        int ret;

        cg_link = container_of(link, struct bpf_cgroup_link, link);

        cgroup_lock();
        /* link might have been auto-released by dying cgroup, so fail */
        if (!cg_link->cgroup) {
                ret = -ENOLINK;
                goto out_unlock;
        }
        if (old_prog && link->prog != old_prog) {
                ret = -EPERM;
                goto out_unlock;
        }
        ret = __cgroup_bpf_replace(cg_link->cgroup, cg_link, new_prog);
out_unlock:
        cgroup_unlock();
        return ret;
}

static struct bpf_prog_list *find_detach_entry(struct hlist_head *progs,
                                               struct bpf_prog *prog,
                                               struct bpf_cgroup_link *link,
                                               bool allow_multi)
{
        struct bpf_prog_list *pl;

        if (!allow_multi) {
                if (hlist_empty(progs))
                        /* report error when trying to detach and nothing is attached */
                        return ERR_PTR(-ENOENT);

                /* to maintain backward compatibility NONE and OVERRIDE cgroups
                 * allow detaching with invalid FD (prog==NULL) in legacy mode
                 */
                return hlist_entry(progs->first, typeof(*pl), node);
        }

        if (!prog && !link)
                /* to detach MULTI prog the user has to specify valid FD
                 * of the program or link to be detached
                 */
                return ERR_PTR(-EINVAL);

        /* find the prog or link and detach it */
        hlist_for_each_entry(pl, progs, node) {
                if (pl->prog == prog && pl->link == link)
                        return pl;
        }
        return ERR_PTR(-ENOENT);
}

/**
 * purge_effective_progs() - After compute_effective_progs fails to alloc new
 *                           cgrp->bpf.inactive table we can recover by
 *                           recomputing the array in place.
 *
 * @cgrp: The cgroup which descendants to travers
 * @prog: A program to detach or NULL
 * @link: A link to detach or NULL
 * @atype: Type of detach operation
 */
static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog *prog,
                                  struct bpf_cgroup_link *link,
                                  enum cgroup_bpf_attach_type atype)
{
        struct cgroup_subsys_state *css;
        struct bpf_prog_array *progs;
        struct bpf_prog_list *pl;
        struct hlist_head *head;
        struct cgroup *cg;
        int pos;

        /* recompute effective prog array in place */
        css_for_each_descendant_pre(css, &cgrp->self) {
                struct cgroup *desc = container_of(css, struct cgroup, self);

                if (percpu_ref_is_zero(&desc->bpf.refcnt))
                        continue;

                /* find position of link or prog in effective progs array */
                for (pos = 0, cg = desc; cg; cg = cgroup_parent(cg)) {
                        if (pos && !(cg->bpf.flags[atype] & BPF_F_ALLOW_MULTI))
                                continue;

                        head = &cg->bpf.progs[atype];
                        hlist_for_each_entry(pl, head, node) {
                                if (!prog_list_prog(pl))
                                        continue;
                                if (pl->prog == prog && pl->link == link)
                                        goto found;
                                pos++;
                        }
                }

                /* no link or prog match, skip the cgroup of this layer */
                continue;
found:
                progs = rcu_dereference_protected(
                                desc->bpf.effective[atype],
                                lockdep_is_held(&cgroup_mutex));

                /* Remove the program from the array */
                WARN_ONCE(bpf_prog_array_delete_safe_at(progs, pos),
                          "Failed to purge a prog from array at index %d", pos);
        }
}

/**
 * __cgroup_bpf_detach() - Detach the program or link from a cgroup, and
 *                         propagate the change to descendants
 * @cgrp: The cgroup which descendants to traverse
 * @prog: A program to detach or NULL
 * @link: A link to detach or NULL
 * @type: Type of detach operation
 *
 * At most one of @prog or @link can be non-NULL.
 * Must be called with cgroup_mutex held.
 */
static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
                               struct bpf_cgroup_link *link, enum bpf_attach_type type)
{
        enum cgroup_bpf_attach_type atype;
        struct bpf_prog *old_prog;
        struct bpf_prog_list *pl;
        struct hlist_head *progs;
        u32 attach_btf_id = 0;
        u32 flags;

        if (prog)
                attach_btf_id = prog->aux->attach_btf_id;
        if (link)
                attach_btf_id = link->link.prog->aux->attach_btf_id;

        atype = bpf_cgroup_atype_find(type, attach_btf_id);
        if (atype < 0)
                return -EINVAL;

        progs = &cgrp->bpf.progs[atype];
        flags = cgrp->bpf.flags[atype];

        if (prog && link)
                /* only one of prog or link can be specified */
                return -EINVAL;

        pl = find_detach_entry(progs, prog, link, flags & BPF_F_ALLOW_MULTI);
        if (IS_ERR(pl))
                return PTR_ERR(pl);

        /* mark it deleted, so it's ignored while recomputing effective */
        old_prog = pl->prog;
        pl->prog = NULL;
        pl->link = NULL;

        if (update_effective_progs(cgrp, atype)) {
                /* if update effective array failed replace the prog with a dummy prog*/
                pl->prog = old_prog;
                pl->link = link;
                purge_effective_progs(cgrp, old_prog, link, atype);
        }

        /* now can actually delete it from this cgroup list */
        hlist_del(&pl->node);

        kfree(pl);
        if (hlist_empty(progs))
                /* last program was detached, reset flags to zero */
                cgrp->bpf.flags[atype] = 0;
        if (old_prog) {
                if (type == BPF_LSM_CGROUP)
                        bpf_trampoline_unlink_cgroup_shim(old_prog);
                bpf_prog_put(old_prog);
        }
        static_branch_dec(&cgroup_bpf_enabled_key[atype]);
        return 0;
}

static int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
                             enum bpf_attach_type type)
{
        int ret;

        cgroup_lock();
        ret = __cgroup_bpf_detach(cgrp, prog, NULL, type);
        cgroup_unlock();
        return ret;
}

/* Must be called with cgroup_mutex held to avoid races. */
static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
                              union bpf_attr __user *uattr)
{
        __u32 __user *prog_attach_flags = u64_to_user_ptr(attr->query.prog_attach_flags);
        bool effective_query = attr->query.query_flags & BPF_F_QUERY_EFFECTIVE;
        __u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids);
        enum bpf_attach_type type = attr->query.attach_type;
        enum cgroup_bpf_attach_type from_atype, to_atype;
        enum cgroup_bpf_attach_type atype;
        struct bpf_prog_array *effective;
        int cnt, ret = 0, i;
        int total_cnt = 0;
        u32 flags;

        if (effective_query && prog_attach_flags)
                return -EINVAL;

        if (type == BPF_LSM_CGROUP) {
                if (!effective_query && attr->query.prog_cnt &&
                    prog_ids && !prog_attach_flags)
                        return -EINVAL;

                from_atype = CGROUP_LSM_START;
                to_atype = CGROUP_LSM_END;
                flags = 0;
        } else {
                from_atype = to_cgroup_bpf_attach_type(type);
                if (from_atype < 0)
                        return -EINVAL;
                to_atype = from_atype;
                flags = cgrp->bpf.flags[from_atype];
        }

        for (atype = from_atype; atype <= to_atype; atype++) {
                if (effective_query) {
                        effective = rcu_dereference_protected(cgrp->bpf.effective[atype],
                                                              lockdep_is_held(&cgroup_mutex));
                        total_cnt += bpf_prog_array_length(effective);
                } else {
                        total_cnt += prog_list_length(&cgrp->bpf.progs[atype], NULL);
                }
        }

        /* always output uattr->query.attach_flags as 0 during effective query */
        flags = effective_query ? 0 : flags;
        if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags)))
                return -EFAULT;
        if (copy_to_user(&uattr->query.prog_cnt, &total_cnt, sizeof(total_cnt)))
                return -EFAULT;
        if (attr->query.prog_cnt == 0 || !prog_ids || !total_cnt)
                /* return early if user requested only program count + flags */
                return 0;

        if (attr->query.prog_cnt < total_cnt) {
                total_cnt = attr->query.prog_cnt;
                ret = -ENOSPC;
        }

        for (atype = from_atype; atype <= to_atype && total_cnt; atype++) {
                if (effective_query) {
                        effective = rcu_dereference_protected(cgrp->bpf.effective[atype],
                                                              lockdep_is_held(&cgroup_mutex));
                        cnt = min_t(int, bpf_prog_array_length(effective), total_cnt);
                        ret = bpf_prog_array_copy_to_user(effective, prog_ids, cnt);
                } else {
                        struct hlist_head *progs;
                        struct bpf_prog_list *pl;
                        struct bpf_prog *prog;
                        u32 id;

                        progs = &cgrp->bpf.progs[atype];
                        cnt = min_t(int, prog_list_length(progs, NULL), total_cnt);
                        i = 0;
                        hlist_for_each_entry(pl, progs, node) {
                                prog = prog_list_prog(pl);
                                id = prog->aux->id;
                                if (copy_to_user(prog_ids + i, &id, sizeof(id)))
                                        return -EFAULT;
                                if (++i == cnt)
                                        break;
                        }

                        if (prog_attach_flags) {
                                flags = cgrp->bpf.flags[atype];

                                for (i = 0; i < cnt; i++)
                                        if (copy_to_user(prog_attach_flags + i,
                                                         &flags, sizeof(flags)))
                                                return -EFAULT;
                                prog_attach_flags += cnt;
                        }
                }

                prog_ids += cnt;
                total_cnt -= cnt;
        }
        return ret;
}

static int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
                            union bpf_attr __user *uattr)
{
        int ret;

        cgroup_lock();
        ret = __cgroup_bpf_query(cgrp, attr, uattr);
        cgroup_unlock();
        return ret;
}

int cgroup_bpf_prog_attach(const union bpf_attr *attr,
                           enum bpf_prog_type ptype, struct bpf_prog *prog)
{
        struct bpf_prog *replace_prog = NULL;
        struct cgroup *cgrp;
        int ret;

        cgrp = cgroup_get_from_fd(attr->target_fd);
        if (IS_ERR(cgrp))
                return PTR_ERR(cgrp);

        if ((attr->attach_flags & BPF_F_ALLOW_MULTI) &&
            (attr->attach_flags & BPF_F_REPLACE)) {
                replace_prog = bpf_prog_get_type(attr->replace_bpf_fd, ptype);
                if (IS_ERR(replace_prog)) {
                        cgroup_put(cgrp);
                        return PTR_ERR(replace_prog);
                }
        }

        ret = cgroup_bpf_attach(cgrp, prog, replace_prog, NULL,
                                attr->attach_type, attr->attach_flags);

        if (replace_prog)
                bpf_prog_put(replace_prog);
        cgroup_put(cgrp);
        return ret;
}

int cgroup_bpf_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype)
{
        struct bpf_prog *prog;
        struct cgroup *cgrp;
        int ret;

        cgrp = cgroup_get_from_fd(attr->target_fd);
        if (IS_ERR(cgrp))
                return PTR_ERR(cgrp);

        prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
        if (IS_ERR(prog))
                prog = NULL;

        ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type);
        if (prog)
                bpf_prog_put(prog);

        cgroup_put(cgrp);
        return ret;
}

static void bpf_cgroup_link_release(struct bpf_link *link)
{
        struct bpf_cgroup_link *cg_link =
                container_of(link, struct bpf_cgroup_link, link);
        struct cgroup *cg;

        /* link might have been auto-detached by dying cgroup already,
         * in that case our work is done here
         */
        if (!cg_link->cgroup)
                return;

        cgroup_lock();

        /* re-check cgroup under lock again */
        if (!cg_link->cgroup) {
                cgroup_unlock();
                return;
        }

        WARN_ON(__cgroup_bpf_detach(cg_link->cgroup, NULL, cg_link,
                                    cg_link->type));
        if (cg_link->type == BPF_LSM_CGROUP)
                bpf_trampoline_unlink_cgroup_shim(cg_link->link.prog);

        cg = cg_link->cgroup;
        cg_link->cgroup = NULL;

        cgroup_unlock();

        cgroup_put(cg);
}

static void bpf_cgroup_link_dealloc(struct bpf_link *link)
{
        struct bpf_cgroup_link *cg_link =
                container_of(link, struct bpf_cgroup_link, link);

        kfree(cg_link);
}

static int bpf_cgroup_link_detach(struct bpf_link *link)
{
        bpf_cgroup_link_release(link);

        return 0;
}

static void bpf_cgroup_link_show_fdinfo(const struct bpf_link *link,
                                        struct seq_file *seq)
{
        struct bpf_cgroup_link *cg_link =
                container_of(link, struct bpf_cgroup_link, link);
        u64 cg_id = 0;

        cgroup_lock();
        if (cg_link->cgroup)
                cg_id = cgroup_id(cg_link->cgroup);
        cgroup_unlock();

        seq_printf(seq,
                   "cgroup_id:\t%llu\n"
                   "attach_type:\t%d\n",
                   cg_id,
                   cg_link->type);
}

static int bpf_cgroup_link_fill_link_info(const struct bpf_link *link,
                                          struct bpf_link_info *info)
{
        struct bpf_cgroup_link *cg_link =
                container_of(link, struct bpf_cgroup_link, link);
        u64 cg_id = 0;

        cgroup_lock();
        if (cg_link->cgroup)
                cg_id = cgroup_id(cg_link->cgroup);
        cgroup_unlock();

        info->cgroup.cgroup_id = cg_id;
        info->cgroup.attach_type = cg_link->type;
        return 0;
}

static const struct bpf_link_ops bpf_cgroup_link_lops = {
        .release = bpf_cgroup_link_release,
        .dealloc = bpf_cgroup_link_dealloc,
        .detach = bpf_cgroup_link_detach,
        .update_prog = cgroup_bpf_replace,
        .show_fdinfo = bpf_cgroup_link_show_fdinfo,
        .fill_link_info = bpf_cgroup_link_fill_link_info,
};

int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
{
        struct bpf_link_primer link_primer;
        struct bpf_cgroup_link *link;
        struct cgroup *cgrp;
        int err;

        if (attr->link_create.flags)
                return -EINVAL;

        cgrp = cgroup_get_from_fd(attr->link_create.target_fd);
        if (IS_ERR(cgrp))
                return PTR_ERR(cgrp);

        link = kzalloc(sizeof(*link), GFP_USER);
        if (!link) {
                err = -ENOMEM;
                goto out_put_cgroup;
        }
        bpf_link_init(&link->link, BPF_LINK_TYPE_CGROUP, &bpf_cgroup_link_lops,
                      prog);
        link->cgroup = cgrp;
        link->type = attr->link_create.attach_type;

        err = bpf_link_prime(&link->link, &link_primer);
        if (err) {
                kfree(link);
                goto out_put_cgroup;
        }

        err = cgroup_bpf_attach(cgrp, NULL, NULL, link,
                                link->type, BPF_F_ALLOW_MULTI);
        if (err) {
                bpf_link_cleanup(&link_primer);
                goto out_put_cgroup;
        }

        return bpf_link_settle(&link_primer);

out_put_cgroup:
        cgroup_put(cgrp);
        return err;
}

int cgroup_bpf_prog_query(const union bpf_attr *attr,
                          union bpf_attr __user *uattr)
{
        struct cgroup *cgrp;
        int ret;

        cgrp = cgroup_get_from_fd(attr->query.target_fd);
        if (IS_ERR(cgrp))
                return PTR_ERR(cgrp);

        ret = cgroup_bpf_query(cgrp, attr, uattr);

        cgroup_put(cgrp);
        return ret;
}

/**
 * __cgroup_bpf_run_filter_skb() - Run a program for packet filtering
 * @sk: The socket sending or receiving traffic
 * @skb: The skb that is being sent or received
 * @atype: The type of program to be executed
 *
 * If no socket is passed, or the socket is not of type INET or INET6,
 * this function does nothing and returns 0.
 *
 * The program type passed in via @type must be suitable for network
 * filtering. No further check is performed to assert that.
 *
 * For egress packets, this function can return:
 *   NET_XMIT_SUCCESS    (0)        - continue with packet output
 *   NET_XMIT_DROP       (1)        - drop packet and notify TCP to call cwr
 *   NET_XMIT_CN         (2)        - continue with packet output and notify TCP
 *                                  to call cwr
 *   -err                        - drop packet
 *
 * For ingress packets, this function will return -EPERM if any
 * attached program was found and if it returned != 1 during execution.
 * Otherwise 0 is returned.
 */
int __cgroup_bpf_run_filter_skb(struct sock *sk,
                                struct sk_buff *skb,
                                enum cgroup_bpf_attach_type atype)
{
        unsigned int offset = -skb_network_offset(skb);
        struct sock *save_sk;
        void *saved_data_end;
        struct cgroup *cgrp;
        int ret;

        if (sk->sk_family != AF_INET && sk->sk_family != AF_INET6)
                return 0;

        cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
        save_sk = skb->sk;
        skb->sk = sk;
        __skb_push(skb, offset);

        /* compute pointers for the bpf prog */
        bpf_compute_and_save_data_end(skb, &saved_data_end);

        if (atype == CGROUP_INET_EGRESS) {
                u32 flags = 0;
                bool cn;

                ret = bpf_prog_run_array_cg(&cgrp->bpf, atype, skb,
                                            __bpf_prog_run_save_cb, 0, &flags);

                /* Return values of CGROUP EGRESS BPF programs are:
                 *   0: drop packet
                 *   1: keep packet
                 *   2: drop packet and cn
                 *   3: keep packet and cn
                 *
                 * The returned value is then converted to one of the NET_XMIT
                 * or an error code that is then interpreted as drop packet
                 * (and no cn):
                 *   0: NET_XMIT_SUCCESS  skb should be transmitted
                 *   1: NET_XMIT_DROP     skb should be dropped and cn
                 *   2: NET_XMIT_CN       skb should be transmitted and cn
                 *   3: -err              skb should be dropped
                 */

                cn = flags & BPF_RET_SET_CN;
                if (ret && !IS_ERR_VALUE((long)ret))
                        ret = -EFAULT;
                if (!ret)
                        ret = (cn ? NET_XMIT_CN : NET_XMIT_SUCCESS);
                else
                        ret = (cn ? NET_XMIT_DROP : ret);
        } else {
                ret = bpf_prog_run_array_cg(&cgrp->bpf, atype,
                                            skb, __bpf_prog_run_save_cb, 0,
                                            NULL);
                if (ret && !IS_ERR_VALUE((long)ret))
                        ret = -EFAULT;
        }
        bpf_restore_data_end(skb, saved_data_end);
        __skb_pull(skb, offset);
        skb->sk = save_sk;

        return ret;
}
EXPORT_SYMBOL(__cgroup_bpf_run_filter_skb);

/**
 * __cgroup_bpf_run_filter_sk() - Run a program on a sock
 * @sk: sock structure to manipulate
 * @atype: The type of program to be executed
 *
 * socket is passed is expected to be of type INET or INET6.
 *
 * The program type passed in via @type must be suitable for sock
 * filtering. No further check is performed to assert that.
 *
 * This function will return %-EPERM if any if an attached program was found
 * and if it returned != 1 during execution. In all other cases, 0 is returned.
 */
int __cgroup_bpf_run_filter_sk(struct sock *sk,
                               enum cgroup_bpf_attach_type atype)
{
        struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);

        return bpf_prog_run_array_cg(&cgrp->bpf, atype, sk, bpf_prog_run, 0,
                                     NULL);
}
EXPORT_SYMBOL(__cgroup_bpf_run_filter_sk);

/**
 * __cgroup_bpf_run_filter_sock_addr() - Run a program on a sock and
 *                                       provided by user sockaddr
 * @sk: sock struct that will use sockaddr
 * @uaddr: sockaddr struct provided by user
 * @uaddrlen: Pointer to the size of the sockaddr struct provided by user. It is
 *            read-only for AF_INET[6] uaddr but can be modified for AF_UNIX
 *            uaddr.
 * @atype: The type of program to be executed
 * @t_ctx: Pointer to attach type specific context
 * @flags: Pointer to u32 which contains higher bits of BPF program
 *         return value (OR'ed together).
 *
 * socket is expected to be of type INET, INET6 or UNIX.
 *
 * This function will return %-EPERM if an attached program is found and
 * returned value != 1 during execution. In all other cases, 0 is returned.
 */
int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,
                                      struct sockaddr *uaddr,
                                      int *uaddrlen,
                                      enum cgroup_bpf_attach_type atype,
                                      void *t_ctx,
                                      u32 *flags)
{
        struct bpf_sock_addr_kern ctx = {
                .sk = sk,
                .uaddr = uaddr,
                .t_ctx = t_ctx,
        };
        struct sockaddr_storage unspec;
        struct cgroup *cgrp;
        int ret;

        /* Check socket family since not all sockets represent network
         * endpoint (e.g. AF_UNIX).
         */
        if (sk->sk_family != AF_INET && sk->sk_family != AF_INET6 &&
            sk->sk_family != AF_UNIX)
                return 0;

        if (!ctx.uaddr) {
                memset(&unspec, 0, sizeof(unspec));
                ctx.uaddr = (struct sockaddr *)&unspec;
                ctx.uaddrlen = 0;
        } else {
                ctx.uaddrlen = *uaddrlen;
        }

        cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
        ret = bpf_prog_run_array_cg(&cgrp->bpf, atype, &ctx, bpf_prog_run,
                                    0, flags);

        if (!ret && uaddr)
                *uaddrlen = ctx.uaddrlen;

        return ret;
}
EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_addr);

/**
 * __cgroup_bpf_run_filter_sock_ops() - Run a program on a sock
 * @sk: socket to get cgroup from
 * @sock_ops: bpf_sock_ops_kern struct to pass to program. Contains
 * sk with connection information (IP addresses, etc.) May not contain
 * cgroup info if it is a req sock.
 * @atype: The type of program to be executed
 *
 * socket passed is expected to be of type INET or INET6.
 *
 * The program type passed in via @type must be suitable for sock_ops
 * filtering. No further check is performed to assert that.
 *
 * This function will return %-EPERM if any if an attached program was found
 * and if it returned != 1 during execution. In all other cases, 0 is returned.
 */
int __cgroup_bpf_run_filter_sock_ops(struct sock *sk,
                                     struct bpf_sock_ops_kern *sock_ops,
                                     enum cgroup_bpf_attach_type atype)
{
        struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);

        return bpf_prog_run_array_cg(&cgrp->bpf, atype, sock_ops, bpf_prog_run,
                                     0, NULL);
}
EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_ops);

int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor,
                                      short access, enum cgroup_bpf_attach_type atype)
{
        struct cgroup *cgrp;
        struct bpf_cgroup_dev_ctx ctx = {
                .access_type = (access << 16) | dev_type,
                .major = major,
                .minor = minor,
        };
        int ret;

        rcu_read_lock();
        cgrp = task_dfl_cgroup(current);
        ret = bpf_prog_run_array_cg(&cgrp->bpf, atype, &ctx, bpf_prog_run, 0,
                                    NULL);
        rcu_read_unlock();

        return ret;
}

BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
{
        /* flags argument is not used now,
         * but provides an ability to extend the API.
         * verifier checks that its value is correct.
         */
        enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
        struct bpf_cgroup_storage *storage;
        struct bpf_cg_run_ctx *ctx;
        void *ptr;

        /* get current cgroup storage from BPF run context */
        ctx = container_of(current->bpf_ctx, struct bpf_cg_run_ctx, run_ctx);
        storage = ctx->prog_item->cgroup_storage[stype];

        if (stype == BPF_CGROUP_STORAGE_SHARED)
                ptr = &READ_ONCE(storage->buf)->data[0];
        else
                ptr = this_cpu_ptr(storage->percpu_buf);

        return (unsigned long)ptr;
}

const struct bpf_func_proto bpf_get_local_storage_proto = {
        .func                = bpf_get_local_storage,
        .gpl_only        = false,
        .ret_type        = RET_PTR_TO_MAP_VALUE,
        .arg1_type        = ARG_CONST_MAP_PTR,
        .arg2_type        = ARG_ANYTHING,
};

BPF_CALL_0(bpf_get_retval)
{
        struct bpf_cg_run_ctx *ctx =
                container_of(current->bpf_ctx, struct bpf_cg_run_ctx, run_ctx);

        return ctx->retval;
}

const struct bpf_func_proto bpf_get_retval_proto = {
        .func                = bpf_get_retval,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
};

BPF_CALL_1(bpf_set_retval, int, retval)
{
        struct bpf_cg_run_ctx *ctx =
                container_of(current->bpf_ctx, struct bpf_cg_run_ctx, run_ctx);

        ctx->retval = retval;
        return 0;
}

const struct bpf_func_proto bpf_set_retval_proto = {
        .func                = bpf_set_retval,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_ANYTHING,
};

static const struct bpf_func_proto *
cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        const struct bpf_func_proto *func_proto;

        func_proto = cgroup_common_func_proto(func_id, prog);
        if (func_proto)
                return func_proto;

        func_proto = cgroup_current_func_proto(func_id, prog);
        if (func_proto)
                return func_proto;

        switch (func_id) {
        case BPF_FUNC_perf_event_output:
                return &bpf_event_output_data_proto;
        default:
                return bpf_base_func_proto(func_id, prog);
        }
}

static bool cgroup_dev_is_valid_access(int off, int size,
                                       enum bpf_access_type type,
                                       const struct bpf_prog *prog,
                                       struct bpf_insn_access_aux *info)
{
        const int size_default = sizeof(__u32);

        if (type == BPF_WRITE)
                return false;

        if (off < 0 || off + size > sizeof(struct bpf_cgroup_dev_ctx))
                return false;
        /* The verifier guarantees that size > 0. */
        if (off % size != 0)
                return false;

        switch (off) {
        case bpf_ctx_range(struct bpf_cgroup_dev_ctx, access_type):
                bpf_ctx_record_field_size(info, size_default);
                if (!bpf_ctx_narrow_access_ok(off, size, size_default))
                        return false;
                break;
        default:
                if (size != size_default)
                        return false;
        }

        return true;
}

const struct bpf_prog_ops cg_dev_prog_ops = {
};

const struct bpf_verifier_ops cg_dev_verifier_ops = {
        .get_func_proto                = cgroup_dev_func_proto,
        .is_valid_access        = cgroup_dev_is_valid_access,
};

/**
 * __cgroup_bpf_run_filter_sysctl - Run a program on sysctl
 *
 * @head: sysctl table header
 * @table: sysctl table
 * @write: sysctl is being read (= 0) or written (= 1)
 * @buf: pointer to buffer (in and out)
 * @pcount: value-result argument: value is size of buffer pointed to by @buf,
 *        result is size of @new_buf if program set new value, initial value
 *        otherwise
 * @ppos: value-result argument: value is position at which read from or write
 *        to sysctl is happening, result is new position if program overrode it,
 *        initial value otherwise
 * @atype: type of program to be executed
 *
 * Program is run when sysctl is being accessed, either read or written, and
 * can allow or deny such access.
 *
 * This function will return %-EPERM if an attached program is found and
 * returned value != 1 during execution. In all other cases 0 is returned.
 */
int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head,
                                   const struct ctl_table *table, int write,
                                   char **buf, size_t *pcount, loff_t *ppos,
                                   enum cgroup_bpf_attach_type atype)
{
        struct bpf_sysctl_kern ctx = {
                .head = head,
                .table = table,
                .write = write,
                .ppos = ppos,
                .cur_val = NULL,
                .cur_len = PAGE_SIZE,
                .new_val = NULL,
                .new_len = 0,
                .new_updated = 0,
        };
        struct cgroup *cgrp;
        loff_t pos = 0;
        int ret;

        ctx.cur_val = kmalloc_track_caller(ctx.cur_len, GFP_KERNEL);
        if (!ctx.cur_val ||
            table->proc_handler(table, 0, ctx.cur_val, &ctx.cur_len, &pos)) {
                /* Let BPF program decide how to proceed. */
                ctx.cur_len = 0;
        }

        if (write && *buf && *pcount) {
                /* BPF program should be able to override new value with a
                 * buffer bigger than provided by user.
                 */
                ctx.new_val = kmalloc_track_caller(PAGE_SIZE, GFP_KERNEL);
                ctx.new_len = min_t(size_t, PAGE_SIZE, *pcount);
                if (ctx.new_val) {
                        memcpy(ctx.new_val, *buf, ctx.new_len);
                } else {
                        /* Let BPF program decide how to proceed. */
                        ctx.new_len = 0;
                }
        }

        rcu_read_lock();
        cgrp = task_dfl_cgroup(current);
        ret = bpf_prog_run_array_cg(&cgrp->bpf, atype, &ctx, bpf_prog_run, 0,
                                    NULL);
        rcu_read_unlock();

        kfree(ctx.cur_val);

        if (ret == 1 && ctx.new_updated) {
                kfree(*buf);
                *buf = ctx.new_val;
                *pcount = ctx.new_len;
        } else {
                kfree(ctx.new_val);
        }

        return ret;
}

#ifdef CONFIG_NET
static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen,
                             struct bpf_sockopt_buf *buf)
{
        if (unlikely(max_optlen < 0))
                return -EINVAL;

        if (unlikely(max_optlen > PAGE_SIZE)) {
                /* We don't expose optvals that are greater than PAGE_SIZE
                 * to the BPF program.
                 */
                max_optlen = PAGE_SIZE;
        }

        if (max_optlen <= sizeof(buf->data)) {
                /* When the optval fits into BPF_SOCKOPT_KERN_BUF_SIZE
                 * bytes avoid the cost of kzalloc.
                 */
                ctx->optval = buf->data;
                ctx->optval_end = ctx->optval + max_optlen;
                return max_optlen;
        }

        ctx->optval = kzalloc(max_optlen, GFP_USER);
        if (!ctx->optval)
                return -ENOMEM;

        ctx->optval_end = ctx->optval + max_optlen;

        return max_optlen;
}

static void sockopt_free_buf(struct bpf_sockopt_kern *ctx,
                             struct bpf_sockopt_buf *buf)
{
        if (ctx->optval == buf->data)
                return;
        kfree(ctx->optval);
}

static bool sockopt_buf_allocated(struct bpf_sockopt_kern *ctx,
                                  struct bpf_sockopt_buf *buf)
{
        return ctx->optval != buf->data;
}

int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level,
                                       int *optname, sockptr_t optval,
                                       int *optlen, char **kernel_optval)
{
        struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
        struct bpf_sockopt_buf buf = {};
        struct bpf_sockopt_kern ctx = {
                .sk = sk,
                .level = *level,
                .optname = *optname,
        };
        int ret, max_optlen;

        /* Allocate a bit more than the initial user buffer for
         * BPF program. The canonical use case is overriding
         * TCP_CONGESTION(nv) to TCP_CONGESTION(cubic).
         */
        max_optlen = max_t(int, 16, *optlen);
        max_optlen = sockopt_alloc_buf(&ctx, max_optlen, &buf);
        if (max_optlen < 0)
                return max_optlen;

        ctx.optlen = *optlen;

        if (copy_from_sockptr(ctx.optval, optval,
                              min(*optlen, max_optlen))) {
                ret = -EFAULT;
                goto out;
        }

        lock_sock(sk);
        ret = bpf_prog_run_array_cg(&cgrp->bpf, CGROUP_SETSOCKOPT,
                                    &ctx, bpf_prog_run, 0, NULL);
        release_sock(sk);

        if (ret)
                goto out;

        if (ctx.optlen == -1) {
                /* optlen set to -1, bypass kernel */
                ret = 1;
        } else if (ctx.optlen > max_optlen || ctx.optlen < -1) {
                /* optlen is out of bounds */
                if (*optlen > PAGE_SIZE && ctx.optlen >= 0) {
                        pr_info_once("bpf setsockopt: ignoring program buffer with optlen=%d (max_optlen=%d)\n",
                                     ctx.optlen, max_optlen);
                        ret = 0;
                        goto out;
                }
                ret = -EFAULT;
        } else {
                /* optlen within bounds, run kernel handler */
                ret = 0;

                /* export any potential modifications */
                *level = ctx.level;
                *optname = ctx.optname;

                /* optlen == 0 from BPF indicates that we should
                 * use original userspace data.
                 */
                if (ctx.optlen != 0) {
                        *optlen = ctx.optlen;
                        /* We've used bpf_sockopt_kern->buf as an intermediary
                         * storage, but the BPF program indicates that we need
                         * to pass this data to the kernel setsockopt handler.
                         * No way to export on-stack buf, have to allocate a
                         * new buffer.
                         */
                        if (!sockopt_buf_allocated(&ctx, &buf)) {
                                void *p = kmalloc(ctx.optlen, GFP_USER);

                                if (!p) {
                                        ret = -ENOMEM;
                                        goto out;
                                }
                                memcpy(p, ctx.optval, ctx.optlen);
                                *kernel_optval = p;
                        } else {
                                *kernel_optval = ctx.optval;
                        }
                        /* export and don't free sockopt buf */
                        return 0;
                }
        }

out:
        sockopt_free_buf(&ctx, &buf);
        return ret;
}

int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level,
                                       int optname, sockptr_t optval,
                                       sockptr_t optlen, int max_optlen,
                                       int retval)
{
        struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
        struct bpf_sockopt_buf buf = {};
        struct bpf_sockopt_kern ctx = {
                .sk = sk,
                .level = level,
                .optname = optname,
                .current_task = current,
        };
        int orig_optlen;
        int ret;

        orig_optlen = max_optlen;
        ctx.optlen = max_optlen;
        max_optlen = sockopt_alloc_buf(&ctx, max_optlen, &buf);
        if (max_optlen < 0)
                return max_optlen;

        if (!retval) {
                /* If kernel getsockopt finished successfully,
                 * copy whatever was returned to the user back
                 * into our temporary buffer. Set optlen to the
                 * one that kernel returned as well to let
                 * BPF programs inspect the value.
                 */
                if (copy_from_sockptr(&ctx.optlen, optlen,
                                      sizeof(ctx.optlen))) {
                        ret = -EFAULT;
                        goto out;
                }

                if (ctx.optlen < 0) {
                        ret = -EFAULT;
                        goto out;
                }
                orig_optlen = ctx.optlen;

                if (copy_from_sockptr(ctx.optval, optval,
                                      min(ctx.optlen, max_optlen))) {
                        ret = -EFAULT;
                        goto out;
                }
        }

        lock_sock(sk);
        ret = bpf_prog_run_array_cg(&cgrp->bpf, CGROUP_GETSOCKOPT,
                                    &ctx, bpf_prog_run, retval, NULL);
        release_sock(sk);

        if (ret < 0)
                goto out;

        if (!sockptr_is_null(optval) &&
            (ctx.optlen > max_optlen || ctx.optlen < 0)) {
                if (orig_optlen > PAGE_SIZE && ctx.optlen >= 0) {
                        pr_info_once("bpf getsockopt: ignoring program buffer with optlen=%d (max_optlen=%d)\n",
                                     ctx.optlen, max_optlen);
                        ret = retval;
                        goto out;
                }
                ret = -EFAULT;
                goto out;
        }

        if (ctx.optlen != 0) {
                if (!sockptr_is_null(optval) &&
                    copy_to_sockptr(optval, ctx.optval, ctx.optlen)) {
                        ret = -EFAULT;
                        goto out;
                }
                if (copy_to_sockptr(optlen, &ctx.optlen, sizeof(ctx.optlen))) {
                        ret = -EFAULT;
                        goto out;
                }
        }

out:
        sockopt_free_buf(&ctx, &buf);
        return ret;
}

int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level,
                                            int optname, void *optval,
                                            int *optlen, int retval)
{
        struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
        struct bpf_sockopt_kern ctx = {
                .sk = sk,
                .level = level,
                .optname = optname,
                .optlen = *optlen,
                .optval = optval,
                .optval_end = optval + *optlen,
                .current_task = current,
        };
        int ret;

        /* Note that __cgroup_bpf_run_filter_getsockopt doesn't copy
         * user data back into BPF buffer when reval != 0. This is
         * done as an optimization to avoid extra copy, assuming
         * kernel won't populate the data in case of an error.
         * Here we always pass the data and memset() should
         * be called if that data shouldn't be "exported".
         */

        ret = bpf_prog_run_array_cg(&cgrp->bpf, CGROUP_GETSOCKOPT,
                                    &ctx, bpf_prog_run, retval, NULL);
        if (ret < 0)
                return ret;

        if (ctx.optlen > *optlen)
                return -EFAULT;

        /* BPF programs can shrink the buffer, export the modifications.
         */
        if (ctx.optlen != 0)
                *optlen = ctx.optlen;

        return ret;
}
#endif

static ssize_t sysctl_cpy_dir(const struct ctl_dir *dir, char **bufp,
                              size_t *lenp)
{
        ssize_t tmp_ret = 0, ret;

        if (dir->header.parent) {
                tmp_ret = sysctl_cpy_dir(dir->header.parent, bufp, lenp);
                if (tmp_ret < 0)
                        return tmp_ret;
        }

        ret = strscpy(*bufp, dir->header.ctl_table[0].procname, *lenp);
        if (ret < 0)
                return ret;
        *bufp += ret;
        *lenp -= ret;
        ret += tmp_ret;

        /* Avoid leading slash. */
        if (!ret)
                return ret;

        tmp_ret = strscpy(*bufp, "/", *lenp);
        if (tmp_ret < 0)
                return tmp_ret;
        *bufp += tmp_ret;
        *lenp -= tmp_ret;

        return ret + tmp_ret;
}

BPF_CALL_4(bpf_sysctl_get_name, struct bpf_sysctl_kern *, ctx, char *, buf,
           size_t, buf_len, u64, flags)
{
        ssize_t tmp_ret = 0, ret;

        if (!buf)
                return -EINVAL;

        if (!(flags & BPF_F_SYSCTL_BASE_NAME)) {
                if (!ctx->head)
                        return -EINVAL;
                tmp_ret = sysctl_cpy_dir(ctx->head->parent, &buf, &buf_len);
                if (tmp_ret < 0)
                        return tmp_ret;
        }

        ret = strscpy(buf, ctx->table->procname, buf_len);

        return ret < 0 ? ret : tmp_ret + ret;
}

static const struct bpf_func_proto bpf_sysctl_get_name_proto = {
        .func                = bpf_sysctl_get_name,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_MEM,
        .arg3_type        = ARG_CONST_SIZE,
        .arg4_type        = ARG_ANYTHING,
};

static int copy_sysctl_value(char *dst, size_t dst_len, char *src,
                             size_t src_len)
{
        if (!dst)
                return -EINVAL;

        if (!dst_len)
                return -E2BIG;

        if (!src || !src_len) {
                memset(dst, 0, dst_len);
                return -EINVAL;
        }

        memcpy(dst, src, min(dst_len, src_len));

        if (dst_len > src_len) {
                memset(dst + src_len, '\0', dst_len - src_len);
                return src_len;
        }

        dst[dst_len - 1] = '\0';

        return -E2BIG;
}

BPF_CALL_3(bpf_sysctl_get_current_value, struct bpf_sysctl_kern *, ctx,
           char *, buf, size_t, buf_len)
{
        return copy_sysctl_value(buf, buf_len, ctx->cur_val, ctx->cur_len);
}

static const struct bpf_func_proto bpf_sysctl_get_current_value_proto = {
        .func                = bpf_sysctl_get_current_value,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg3_type        = ARG_CONST_SIZE,
};

BPF_CALL_3(bpf_sysctl_get_new_value, struct bpf_sysctl_kern *, ctx, char *, buf,
           size_t, buf_len)
{
        if (!ctx->write) {
                if (buf && buf_len)
                        memset(buf, '\0', buf_len);
                return -EINVAL;
        }
        return copy_sysctl_value(buf, buf_len, ctx->new_val, ctx->new_len);
}

static const struct bpf_func_proto bpf_sysctl_get_new_value_proto = {
        .func                = bpf_sysctl_get_new_value,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg3_type        = ARG_CONST_SIZE,
};

BPF_CALL_3(bpf_sysctl_set_new_value, struct bpf_sysctl_kern *, ctx,
           const char *, buf, size_t, buf_len)
{
        if (!ctx->write || !ctx->new_val || !ctx->new_len || !buf || !buf_len)
                return -EINVAL;

        if (buf_len > PAGE_SIZE - 1)
                return -E2BIG;

        memcpy(ctx->new_val, buf, buf_len);
        ctx->new_len = buf_len;
        ctx->new_updated = 1;

        return 0;
}

static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = {
        .func                = bpf_sysctl_set_new_value,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type        = ARG_CONST_SIZE,
};

static const struct bpf_func_proto *
sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        const struct bpf_func_proto *func_proto;

        func_proto = cgroup_common_func_proto(func_id, prog);
        if (func_proto)
                return func_proto;

        func_proto = cgroup_current_func_proto(func_id, prog);
        if (func_proto)
                return func_proto;

        switch (func_id) {
        case BPF_FUNC_sysctl_get_name:
                return &bpf_sysctl_get_name_proto;
        case BPF_FUNC_sysctl_get_current_value:
                return &bpf_sysctl_get_current_value_proto;
        case BPF_FUNC_sysctl_get_new_value:
                return &bpf_sysctl_get_new_value_proto;
        case BPF_FUNC_sysctl_set_new_value:
                return &bpf_sysctl_set_new_value_proto;
        case BPF_FUNC_ktime_get_coarse_ns:
                return &bpf_ktime_get_coarse_ns_proto;
        case BPF_FUNC_perf_event_output:
                return &bpf_event_output_data_proto;
        default:
                return bpf_base_func_proto(func_id, prog);
        }
}

static bool sysctl_is_valid_access(int off, int size, enum bpf_access_type type,
                                   const struct bpf_prog *prog,
                                   struct bpf_insn_access_aux *info)
{
        const int size_default = sizeof(__u32);

        if (off < 0 || off + size > sizeof(struct bpf_sysctl) || off % size)
                return false;

        switch (off) {
        case bpf_ctx_range(struct bpf_sysctl, write):
                if (type != BPF_READ)
                        return false;
                bpf_ctx_record_field_size(info, size_default);
                return bpf_ctx_narrow_access_ok(off, size, size_default);
        case bpf_ctx_range(struct bpf_sysctl, file_pos):
                if (type == BPF_READ) {
                        bpf_ctx_record_field_size(info, size_default);
                        return bpf_ctx_narrow_access_ok(off, size, size_default);
                } else {
                        return size == size_default;
                }
        default:
                return false;
        }
}

static u32 sysctl_convert_ctx_access(enum bpf_access_type type,
                                     const struct bpf_insn *si,
                                     struct bpf_insn *insn_buf,
                                     struct bpf_prog *prog, u32 *target_size)
{
        struct bpf_insn *insn = insn_buf;
        u32 read_size;

        switch (si->off) {
        case offsetof(struct bpf_sysctl, write):
                *insn++ = BPF_LDX_MEM(
                        BPF_SIZE(si->code), si->dst_reg, si->src_reg,
                        bpf_target_off(struct bpf_sysctl_kern, write,
                                       sizeof_field(struct bpf_sysctl_kern,
                                                    write),
                                       target_size));
                break;
        case offsetof(struct bpf_sysctl, file_pos):
                /* ppos is a pointer so it should be accessed via indirect
                 * loads and stores. Also for stores additional temporary
                 * register is used since neither src_reg nor dst_reg can be
                 * overridden.
                 */
                if (type == BPF_WRITE) {
                        int treg = BPF_REG_9;

                        if (si->src_reg == treg || si->dst_reg == treg)
                                --treg;
                        if (si->src_reg == treg || si->dst_reg == treg)
                                --treg;
                        *insn++ = BPF_STX_MEM(
                                BPF_DW, si->dst_reg, treg,
                                offsetof(struct bpf_sysctl_kern, tmp_reg));
                        *insn++ = BPF_LDX_MEM(
                                BPF_FIELD_SIZEOF(struct bpf_sysctl_kern, ppos),
                                treg, si->dst_reg,
                                offsetof(struct bpf_sysctl_kern, ppos));
                        *insn++ = BPF_RAW_INSN(
                                BPF_CLASS(si->code) | BPF_MEM | BPF_SIZEOF(u32),
                                treg, si->src_reg,
                                bpf_ctx_narrow_access_offset(
                                        0, sizeof(u32), sizeof(loff_t)),
                                si->imm);
                        *insn++ = BPF_LDX_MEM(
                                BPF_DW, treg, si->dst_reg,
                                offsetof(struct bpf_sysctl_kern, tmp_reg));
                } else {
                        *insn++ = BPF_LDX_MEM(
                                BPF_FIELD_SIZEOF(struct bpf_sysctl_kern, ppos),
                                si->dst_reg, si->src_reg,
                                offsetof(struct bpf_sysctl_kern, ppos));
                        read_size = bpf_size_to_bytes(BPF_SIZE(si->code));
                        *insn++ = BPF_LDX_MEM(
                                BPF_SIZE(si->code), si->dst_reg, si->dst_reg,
                                bpf_ctx_narrow_access_offset(
                                        0, read_size, sizeof(loff_t)));
                }
                *target_size = sizeof(u32);
                break;
        }

        return insn - insn_buf;
}

const struct bpf_verifier_ops cg_sysctl_verifier_ops = {
        .get_func_proto                = sysctl_func_proto,
        .is_valid_access        = sysctl_is_valid_access,
        .convert_ctx_access        = sysctl_convert_ctx_access,
};

const struct bpf_prog_ops cg_sysctl_prog_ops = {
};

#ifdef CONFIG_NET
BPF_CALL_1(bpf_get_netns_cookie_sockopt, struct bpf_sockopt_kern *, ctx)
{
        const struct net *net = ctx ? sock_net(ctx->sk) : &init_net;

        return net->net_cookie;
}

static const struct bpf_func_proto bpf_get_netns_cookie_sockopt_proto = {
        .func                = bpf_get_netns_cookie_sockopt,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX_OR_NULL,
};
#endif

static const struct bpf_func_proto *
cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        const struct bpf_func_proto *func_proto;

        func_proto = cgroup_common_func_proto(func_id, prog);
        if (func_proto)
                return func_proto;

        func_proto = cgroup_current_func_proto(func_id, prog);
        if (func_proto)
                return func_proto;

        switch (func_id) {
#ifdef CONFIG_NET
        case BPF_FUNC_get_netns_cookie:
                return &bpf_get_netns_cookie_sockopt_proto;
        case BPF_FUNC_sk_storage_get:
                return &bpf_sk_storage_get_proto;
        case BPF_FUNC_sk_storage_delete:
                return &bpf_sk_storage_delete_proto;
        case BPF_FUNC_setsockopt:
                if (prog->expected_attach_type == BPF_CGROUP_SETSOCKOPT)
                        return &bpf_sk_setsockopt_proto;
                return NULL;
        case BPF_FUNC_getsockopt:
                if (prog->expected_attach_type == BPF_CGROUP_SETSOCKOPT)
                        return &bpf_sk_getsockopt_proto;
                return NULL;
#endif
#ifdef CONFIG_INET
        case BPF_FUNC_tcp_sock:
                return &bpf_tcp_sock_proto;
#endif
        case BPF_FUNC_perf_event_output:
                return &bpf_event_output_data_proto;
        default:
                return bpf_base_func_proto(func_id, prog);
        }
}

static bool cg_sockopt_is_valid_access(int off, int size,
                                       enum bpf_access_type type,
                                       const struct bpf_prog *prog,
                                       struct bpf_insn_access_aux *info)
{
        const int size_default = sizeof(__u32);

        if (off < 0 || off >= sizeof(struct bpf_sockopt))
                return false;

        if (off % size != 0)
                return false;

        if (type == BPF_WRITE) {
                switch (off) {
                case offsetof(struct bpf_sockopt, retval):
                        if (size != size_default)
                                return false;
                        return prog->expected_attach_type ==
                                BPF_CGROUP_GETSOCKOPT;
                case offsetof(struct bpf_sockopt, optname):
                        fallthrough;
                case offsetof(struct bpf_sockopt, level):
                        if (size != size_default)
                                return false;
                        return prog->expected_attach_type ==
                                BPF_CGROUP_SETSOCKOPT;
                case offsetof(struct bpf_sockopt, optlen):
                        return size == size_default;
                default:
                        return false;
                }
        }

        switch (off) {
        case offsetof(struct bpf_sockopt, sk):
                if (size != sizeof(__u64))
                        return false;
                info->reg_type = PTR_TO_SOCKET;
                break;
        case offsetof(struct bpf_sockopt, optval):
                if (size != sizeof(__u64))
                        return false;
                info->reg_type = PTR_TO_PACKET;
                break;
        case offsetof(struct bpf_sockopt, optval_end):
                if (size != sizeof(__u64))
                        return false;
                info->reg_type = PTR_TO_PACKET_END;
                break;
        case offsetof(struct bpf_sockopt, retval):
                if (size != size_default)
                        return false;
                return prog->expected_attach_type == BPF_CGROUP_GETSOCKOPT;
        default:
                if (size != size_default)
                        return false;
                break;
        }
        return true;
}

#define CG_SOCKOPT_READ_FIELD(F)                                        \
        BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sockopt_kern, F),        \
                    si->dst_reg, si->src_reg,                                \
                    offsetof(struct bpf_sockopt_kern, F))

#define CG_SOCKOPT_WRITE_FIELD(F)                                        \
        BPF_RAW_INSN((BPF_FIELD_SIZEOF(struct bpf_sockopt_kern, F) |        \
                      BPF_MEM | BPF_CLASS(si->code)),                        \
                     si->dst_reg, si->src_reg,                                \
                     offsetof(struct bpf_sockopt_kern, F),                \
                     si->imm)

static u32 cg_sockopt_convert_ctx_access(enum bpf_access_type type,
                                         const struct bpf_insn *si,
                                         struct bpf_insn *insn_buf,
                                         struct bpf_prog *prog,
                                         u32 *target_size)
{
        struct bpf_insn *insn = insn_buf;

        switch (si->off) {
        case offsetof(struct bpf_sockopt, sk):
                *insn++ = CG_SOCKOPT_READ_FIELD(sk);
                break;
        case offsetof(struct bpf_sockopt, level):
                if (type == BPF_WRITE)
                        *insn++ = CG_SOCKOPT_WRITE_FIELD(level);
                else
                        *insn++ = CG_SOCKOPT_READ_FIELD(level);
                break;
        case offsetof(struct bpf_sockopt, optname):
                if (type == BPF_WRITE)
                        *insn++ = CG_SOCKOPT_WRITE_FIELD(optname);
                else
                        *insn++ = CG_SOCKOPT_READ_FIELD(optname);
                break;
        case offsetof(struct bpf_sockopt, optlen):
                if (type == BPF_WRITE)
                        *insn++ = CG_SOCKOPT_WRITE_FIELD(optlen);
                else
                        *insn++ = CG_SOCKOPT_READ_FIELD(optlen);
                break;
        case offsetof(struct bpf_sockopt, retval):
                BUILD_BUG_ON(offsetof(struct bpf_cg_run_ctx, run_ctx) != 0);

                if (type == BPF_WRITE) {
                        int treg = BPF_REG_9;

                        if (si->src_reg == treg || si->dst_reg == treg)
                                --treg;
                        if (si->src_reg == treg || si->dst_reg == treg)
                                --treg;
                        *insn++ = BPF_STX_MEM(BPF_DW, si->dst_reg, treg,
                                              offsetof(struct bpf_sockopt_kern, tmp_reg));
                        *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sockopt_kern, current_task),
                                              treg, si->dst_reg,
                                              offsetof(struct bpf_sockopt_kern, current_task));
                        *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct task_struct, bpf_ctx),
                                              treg, treg,
                                              offsetof(struct task_struct, bpf_ctx));
                        *insn++ = BPF_RAW_INSN(BPF_CLASS(si->code) | BPF_MEM |
                                               BPF_FIELD_SIZEOF(struct bpf_cg_run_ctx, retval),
                                               treg, si->src_reg,
                                               offsetof(struct bpf_cg_run_ctx, retval),
                                               si->imm);
                        *insn++ = BPF_LDX_MEM(BPF_DW, treg, si->dst_reg,
                                              offsetof(struct bpf_sockopt_kern, tmp_reg));
                } else {
                        *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sockopt_kern, current_task),
                                              si->dst_reg, si->src_reg,
                                              offsetof(struct bpf_sockopt_kern, current_task));
                        *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct task_struct, bpf_ctx),
                                              si->dst_reg, si->dst_reg,
                                              offsetof(struct task_struct, bpf_ctx));
                        *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_cg_run_ctx, retval),
                                              si->dst_reg, si->dst_reg,
                                              offsetof(struct bpf_cg_run_ctx, retval));
                }
                break;
        case offsetof(struct bpf_sockopt, optval):
                *insn++ = CG_SOCKOPT_READ_FIELD(optval);
                break;
        case offsetof(struct bpf_sockopt, optval_end):
                *insn++ = CG_SOCKOPT_READ_FIELD(optval_end);
                break;
        }

        return insn - insn_buf;
}

static int cg_sockopt_get_prologue(struct bpf_insn *insn_buf,
                                   bool direct_write,
                                   const struct bpf_prog *prog)
{
        /* Nothing to do for sockopt argument. The data is kzalloc'ated.
         */
        return 0;
}

const struct bpf_verifier_ops cg_sockopt_verifier_ops = {
        .get_func_proto                = cg_sockopt_func_proto,
        .is_valid_access        = cg_sockopt_is_valid_access,
        .convert_ctx_access        = cg_sockopt_convert_ctx_access,
        .gen_prologue                = cg_sockopt_get_prologue,
};

const struct bpf_prog_ops cg_sockopt_prog_ops = {
};

/* Common helpers for cgroup hooks. */
const struct bpf_func_proto *
cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        switch (func_id) {
        case BPF_FUNC_get_local_storage:
                return &bpf_get_local_storage_proto;
        case BPF_FUNC_get_retval:
                switch (prog->expected_attach_type) {
                case BPF_CGROUP_INET_INGRESS:
                case BPF_CGROUP_INET_EGRESS:
                case BPF_CGROUP_SOCK_OPS:
                case BPF_CGROUP_UDP4_RECVMSG:
                case BPF_CGROUP_UDP6_RECVMSG:
                case BPF_CGROUP_UNIX_RECVMSG:
                case BPF_CGROUP_INET4_GETPEERNAME:
                case BPF_CGROUP_INET6_GETPEERNAME:
                case BPF_CGROUP_UNIX_GETPEERNAME:
                case BPF_CGROUP_INET4_GETSOCKNAME:
                case BPF_CGROUP_INET6_GETSOCKNAME:
                case BPF_CGROUP_UNIX_GETSOCKNAME:
                        return NULL;
                default:
                        return &bpf_get_retval_proto;
                }
        case BPF_FUNC_set_retval:
                switch (prog->expected_attach_type) {
                case BPF_CGROUP_INET_INGRESS:
                case BPF_CGROUP_INET_EGRESS:
                case BPF_CGROUP_SOCK_OPS:
                case BPF_CGROUP_UDP4_RECVMSG:
                case BPF_CGROUP_UDP6_RECVMSG:
                case BPF_CGROUP_UNIX_RECVMSG:
                case BPF_CGROUP_INET4_GETPEERNAME:
                case BPF_CGROUP_INET6_GETPEERNAME:
                case BPF_CGROUP_UNIX_GETPEERNAME:
                case BPF_CGROUP_INET4_GETSOCKNAME:
                case BPF_CGROUP_INET6_GETSOCKNAME:
                case BPF_CGROUP_UNIX_GETSOCKNAME:
                        return NULL;
                default:
                        return &bpf_set_retval_proto;
                }
        default:
                return NULL;
        }
}

/* Common helpers for cgroup hooks with valid process context. */
const struct bpf_func_proto *
cgroup_current_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        switch (func_id) {
        case BPF_FUNC_get_current_uid_gid:
                return &bpf_get_current_uid_gid_proto;
        case BPF_FUNC_get_current_comm:
                return &bpf_get_current_comm_proto;
#ifdef CONFIG_CGROUP_NET_CLASSID
        case BPF_FUNC_get_cgroup_classid:
                return &bpf_get_cgroup_classid_curr_proto;
#endif
        case BPF_FUNC_current_task_under_cgroup:
                return &bpf_current_task_under_cgroup_proto;
        default:
                return NULL;
        }
}

















































    1 




   13 





    7 
    1 



    6 







    1 














    5 






    4 

    2 




















    8 
    8 
















    8 
    8 




    8 

    8 















    8 
    8 



























   13 











   13 
































   13 

   13 































































































































































































    8 



    8 



















    8 


    8 




































































































































































































































    4 






    3 





    1 


















    5 





    5 






    2 



    3 




    3 



    3 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
// SPDX-License-Identifier: GPL-2.0
/*
 * To speed up listener socket lookup, create an array to store all sockets
 * listening on the same port.  This allows a decision to be made after finding
 * the first socket.  An optional BPF program can also be configured for
 * selecting the socket index from the array of available sockets.
 */

#include <net/ip.h>
#include <net/sock_reuseport.h>
#include <linux/bpf.h>
#include <linux/idr.h>
#include <linux/filter.h>
#include <linux/rcupdate.h>

#define INIT_SOCKS 128

DEFINE_SPINLOCK(reuseport_lock);

static DEFINE_IDA(reuseport_ida);
static int reuseport_resurrect(struct sock *sk, struct sock_reuseport *old_reuse,
                               struct sock_reuseport *reuse, bool bind_inany);

void reuseport_has_conns_set(struct sock *sk)
{
        struct sock_reuseport *reuse;

        if (!rcu_access_pointer(sk->sk_reuseport_cb))
                return;

        spin_lock_bh(&reuseport_lock);
        reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
                                          lockdep_is_held(&reuseport_lock));
        if (likely(reuse))
                reuse->has_conns = 1;
        spin_unlock_bh(&reuseport_lock);
}
EXPORT_SYMBOL(reuseport_has_conns_set);

static void __reuseport_get_incoming_cpu(struct sock_reuseport *reuse)
{
        /* Paired with READ_ONCE() in reuseport_select_sock_by_hash(). */
        WRITE_ONCE(reuse->incoming_cpu, reuse->incoming_cpu + 1);
}

static void __reuseport_put_incoming_cpu(struct sock_reuseport *reuse)
{
        /* Paired with READ_ONCE() in reuseport_select_sock_by_hash(). */
        WRITE_ONCE(reuse->incoming_cpu, reuse->incoming_cpu - 1);
}

static void reuseport_get_incoming_cpu(struct sock *sk, struct sock_reuseport *reuse)
{
        if (sk->sk_incoming_cpu >= 0)
                __reuseport_get_incoming_cpu(reuse);
}

static void reuseport_put_incoming_cpu(struct sock *sk, struct sock_reuseport *reuse)
{
        if (sk->sk_incoming_cpu >= 0)
                __reuseport_put_incoming_cpu(reuse);
}

void reuseport_update_incoming_cpu(struct sock *sk, int val)
{
        struct sock_reuseport *reuse;
        int old_sk_incoming_cpu;

        if (unlikely(!rcu_access_pointer(sk->sk_reuseport_cb))) {
                /* Paired with REAE_ONCE() in sk_incoming_cpu_update()
                 * and compute_score().
                 */
                WRITE_ONCE(sk->sk_incoming_cpu, val);
                return;
        }

        spin_lock_bh(&reuseport_lock);

        /* This must be done under reuseport_lock to avoid a race with
         * reuseport_grow(), which accesses sk->sk_incoming_cpu without
         * lock_sock() when detaching a shutdown()ed sk.
         *
         * Paired with READ_ONCE() in reuseport_select_sock_by_hash().
         */
        old_sk_incoming_cpu = sk->sk_incoming_cpu;
        WRITE_ONCE(sk->sk_incoming_cpu, val);

        reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
                                          lockdep_is_held(&reuseport_lock));

        /* reuseport_grow() has detached a closed sk. */
        if (!reuse)
                goto out;

        if (old_sk_incoming_cpu < 0 && val >= 0)
                __reuseport_get_incoming_cpu(reuse);
        else if (old_sk_incoming_cpu >= 0 && val < 0)
                __reuseport_put_incoming_cpu(reuse);

out:
        spin_unlock_bh(&reuseport_lock);
}

static int reuseport_sock_index(struct sock *sk,
                                const struct sock_reuseport *reuse,
                                bool closed)
{
        int left, right;

        if (!closed) {
                left = 0;
                right = reuse->num_socks;
        } else {
                left = reuse->max_socks - reuse->num_closed_socks;
                right = reuse->max_socks;
        }

        for (; left < right; left++)
                if (reuse->socks[left] == sk)
                        return left;
        return -1;
}

static void __reuseport_add_sock(struct sock *sk,
                                 struct sock_reuseport *reuse)
{
        reuse->socks[reuse->num_socks] = sk;
        /* paired with smp_rmb() in reuseport_(select|migrate)_sock() */
        smp_wmb();
        reuse->num_socks++;
        reuseport_get_incoming_cpu(sk, reuse);
}

static bool __reuseport_detach_sock(struct sock *sk,
                                    struct sock_reuseport *reuse)
{
        int i = reuseport_sock_index(sk, reuse, false);

        if (i == -1)
                return false;

        reuse->socks[i] = reuse->socks[reuse->num_socks - 1];
        reuse->num_socks--;
        reuseport_put_incoming_cpu(sk, reuse);

        return true;
}

static void __reuseport_add_closed_sock(struct sock *sk,
                                        struct sock_reuseport *reuse)
{
        reuse->socks[reuse->max_socks - reuse->num_closed_socks - 1] = sk;
        /* paired with READ_ONCE() in inet_csk_bind_conflict() */
        WRITE_ONCE(reuse->num_closed_socks, reuse->num_closed_socks + 1);
        reuseport_get_incoming_cpu(sk, reuse);
}

static bool __reuseport_detach_closed_sock(struct sock *sk,
                                           struct sock_reuseport *reuse)
{
        int i = reuseport_sock_index(sk, reuse, true);

        if (i == -1)
                return false;

        reuse->socks[i] = reuse->socks[reuse->max_socks - reuse->num_closed_socks];
        /* paired with READ_ONCE() in inet_csk_bind_conflict() */
        WRITE_ONCE(reuse->num_closed_socks, reuse->num_closed_socks - 1);
        reuseport_put_incoming_cpu(sk, reuse);

        return true;
}

static struct sock_reuseport *__reuseport_alloc(unsigned int max_socks)
{
        struct sock_reuseport *reuse;

        reuse = kzalloc(struct_size(reuse, socks, max_socks), GFP_ATOMIC);
        if (!reuse)
                return NULL;

        reuse->max_socks = max_socks;

        RCU_INIT_POINTER(reuse->prog, NULL);
        return reuse;
}

int reuseport_alloc(struct sock *sk, bool bind_inany)
{
        struct sock_reuseport *reuse;
        int id, ret = 0;

        /* bh lock used since this function call may precede hlist lock in
         * soft irq of receive path or setsockopt from process context
         */
        spin_lock_bh(&reuseport_lock);

        /* Allocation attempts can occur concurrently via the setsockopt path
         * and the bind/hash path.  Nothing to do when we lose the race.
         */
        reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
                                          lockdep_is_held(&reuseport_lock));
        if (reuse) {
                if (reuse->num_closed_socks) {
                        /* sk was shutdown()ed before */
                        ret = reuseport_resurrect(sk, reuse, NULL, bind_inany);
                        goto out;
                }

                /* Only set reuse->bind_inany if the bind_inany is true.
                 * Otherwise, it will overwrite the reuse->bind_inany
                 * which was set by the bind/hash path.
                 */
                if (bind_inany)
                        reuse->bind_inany = bind_inany;
                goto out;
        }

        reuse = __reuseport_alloc(INIT_SOCKS);
        if (!reuse) {
                ret = -ENOMEM;
                goto out;
        }

        id = ida_alloc(&reuseport_ida, GFP_ATOMIC);
        if (id < 0) {
                kfree(reuse);
                ret = id;
                goto out;
        }

        reuse->reuseport_id = id;
        reuse->bind_inany = bind_inany;
        reuse->socks[0] = sk;
        reuse->num_socks = 1;
        reuseport_get_incoming_cpu(sk, reuse);
        rcu_assign_pointer(sk->sk_reuseport_cb, reuse);

out:
        spin_unlock_bh(&reuseport_lock);

        return ret;
}
EXPORT_SYMBOL(reuseport_alloc);

static struct sock_reuseport *reuseport_grow(struct sock_reuseport *reuse)
{
        struct sock_reuseport *more_reuse;
        u32 more_socks_size, i;

        more_socks_size = reuse->max_socks * 2U;
        if (more_socks_size > U16_MAX) {
                if (reuse->num_closed_socks) {
                        /* Make room by removing a closed sk.
                         * The child has already been migrated.
                         * Only reqsk left at this point.
                         */
                        struct sock *sk;

                        sk = reuse->socks[reuse->max_socks - reuse->num_closed_socks];
                        RCU_INIT_POINTER(sk->sk_reuseport_cb, NULL);
                        __reuseport_detach_closed_sock(sk, reuse);

                        return reuse;
                }

                return NULL;
        }

        more_reuse = __reuseport_alloc(more_socks_size);
        if (!more_reuse)
                return NULL;

        more_reuse->num_socks = reuse->num_socks;
        more_reuse->num_closed_socks = reuse->num_closed_socks;
        more_reuse->prog = reuse->prog;
        more_reuse->reuseport_id = reuse->reuseport_id;
        more_reuse->bind_inany = reuse->bind_inany;
        more_reuse->has_conns = reuse->has_conns;
        more_reuse->incoming_cpu = reuse->incoming_cpu;

        memcpy(more_reuse->socks, reuse->socks,
               reuse->num_socks * sizeof(struct sock *));
        memcpy(more_reuse->socks +
               (more_reuse->max_socks - more_reuse->num_closed_socks),
               reuse->socks + (reuse->max_socks - reuse->num_closed_socks),
               reuse->num_closed_socks * sizeof(struct sock *));
        more_reuse->synq_overflow_ts = READ_ONCE(reuse->synq_overflow_ts);

        for (i = 0; i < reuse->max_socks; ++i)
                rcu_assign_pointer(reuse->socks[i]->sk_reuseport_cb,
                                   more_reuse);

        /* Note: we use kfree_rcu here instead of reuseport_free_rcu so
         * that reuse and more_reuse can temporarily share a reference
         * to prog.
         */
        kfree_rcu(reuse, rcu);
        return more_reuse;
}

static void reuseport_free_rcu(struct rcu_head *head)
{
        struct sock_reuseport *reuse;

        reuse = container_of(head, struct sock_reuseport, rcu);
        sk_reuseport_prog_free(rcu_dereference_protected(reuse->prog, 1));
        ida_free(&reuseport_ida, reuse->reuseport_id);
        kfree(reuse);
}

/**
 *  reuseport_add_sock - Add a socket to the reuseport group of another.
 *  @sk:  New socket to add to the group.
 *  @sk2: Socket belonging to the existing reuseport group.
 *  @bind_inany: Whether or not the group is bound to a local INANY address.
 *
 *  May return ENOMEM and not add socket to group under memory pressure.
 */
int reuseport_add_sock(struct sock *sk, struct sock *sk2, bool bind_inany)
{
        struct sock_reuseport *old_reuse, *reuse;

        if (!rcu_access_pointer(sk2->sk_reuseport_cb)) {
                int err = reuseport_alloc(sk2, bind_inany);

                if (err)
                        return err;
        }

        spin_lock_bh(&reuseport_lock);
        reuse = rcu_dereference_protected(sk2->sk_reuseport_cb,
                                          lockdep_is_held(&reuseport_lock));
        old_reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
                                              lockdep_is_held(&reuseport_lock));
        if (old_reuse && old_reuse->num_closed_socks) {
                /* sk was shutdown()ed before */
                int err = reuseport_resurrect(sk, old_reuse, reuse, reuse->bind_inany);

                spin_unlock_bh(&reuseport_lock);
                return err;
        }

        if (old_reuse && old_reuse->num_socks != 1) {
                spin_unlock_bh(&reuseport_lock);
                return -EBUSY;
        }

        if (reuse->num_socks + reuse->num_closed_socks == reuse->max_socks) {
                reuse = reuseport_grow(reuse);
                if (!reuse) {
                        spin_unlock_bh(&reuseport_lock);
                        return -ENOMEM;
                }
        }

        __reuseport_add_sock(sk, reuse);
        rcu_assign_pointer(sk->sk_reuseport_cb, reuse);

        spin_unlock_bh(&reuseport_lock);

        if (old_reuse)
                call_rcu(&old_reuse->rcu, reuseport_free_rcu);
        return 0;
}
EXPORT_SYMBOL(reuseport_add_sock);

static int reuseport_resurrect(struct sock *sk, struct sock_reuseport *old_reuse,
                               struct sock_reuseport *reuse, bool bind_inany)
{
        if (old_reuse == reuse) {
                /* If sk was in the same reuseport group, just pop sk out of
                 * the closed section and push sk into the listening section.
                 */
                __reuseport_detach_closed_sock(sk, old_reuse);
                __reuseport_add_sock(sk, old_reuse);
                return 0;
        }

        if (!reuse) {
                /* In bind()/listen() path, we cannot carry over the eBPF prog
                 * for the shutdown()ed socket. In setsockopt() path, we should
                 * not change the eBPF prog of listening sockets by attaching a
                 * prog to the shutdown()ed socket. Thus, we will allocate a new
                 * reuseport group and detach sk from the old group.
                 */
                int id;

                reuse = __reuseport_alloc(INIT_SOCKS);
                if (!reuse)
                        return -ENOMEM;

                id = ida_alloc(&reuseport_ida, GFP_ATOMIC);
                if (id < 0) {
                        kfree(reuse);
                        return id;
                }

                reuse->reuseport_id = id;
                reuse->bind_inany = bind_inany;
        } else {
                /* Move sk from the old group to the new one if
                 * - all the other listeners in the old group were close()d or
                 *   shutdown()ed, and then sk2 has listen()ed on the same port
                 * OR
                 * - sk listen()ed without bind() (or with autobind), was
                 *   shutdown()ed, and then listen()s on another port which
                 *   sk2 listen()s on.
                 */
                if (reuse->num_socks + reuse->num_closed_socks == reuse->max_socks) {
                        reuse = reuseport_grow(reuse);
                        if (!reuse)
                                return -ENOMEM;
                }
        }

        __reuseport_detach_closed_sock(sk, old_reuse);
        __reuseport_add_sock(sk, reuse);
        rcu_assign_pointer(sk->sk_reuseport_cb, reuse);

        if (old_reuse->num_socks + old_reuse->num_closed_socks == 0)
                call_rcu(&old_reuse->rcu, reuseport_free_rcu);

        return 0;
}

void reuseport_detach_sock(struct sock *sk)
{
        struct sock_reuseport *reuse;

        spin_lock_bh(&reuseport_lock);
        reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
                                          lockdep_is_held(&reuseport_lock));

        /* reuseport_grow() has detached a closed sk */
        if (!reuse)
                goto out;

        /* Notify the bpf side. The sk may be added to a sockarray
         * map. If so, sockarray logic will remove it from the map.
         *
         * Other bpf map types that work with reuseport, like sockmap,
         * don't need an explicit callback from here. They override sk
         * unhash/close ops to remove the sk from the map before we
         * get to this point.
         */
        bpf_sk_reuseport_detach(sk);

        rcu_assign_pointer(sk->sk_reuseport_cb, NULL);

        if (!__reuseport_detach_closed_sock(sk, reuse))
                __reuseport_detach_sock(sk, reuse);

        if (reuse->num_socks + reuse->num_closed_socks == 0)
                call_rcu(&reuse->rcu, reuseport_free_rcu);

out:
        spin_unlock_bh(&reuseport_lock);
}
EXPORT_SYMBOL(reuseport_detach_sock);

void reuseport_stop_listen_sock(struct sock *sk)
{
        if (sk->sk_protocol == IPPROTO_TCP) {
                struct sock_reuseport *reuse;
                struct bpf_prog *prog;

                spin_lock_bh(&reuseport_lock);

                reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
                                                  lockdep_is_held(&reuseport_lock));
                prog = rcu_dereference_protected(reuse->prog,
                                                 lockdep_is_held(&reuseport_lock));

                if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_migrate_req) ||
                    (prog && prog->expected_attach_type == BPF_SK_REUSEPORT_SELECT_OR_MIGRATE)) {
                        /* Migration capable, move sk from the listening section
                         * to the closed section.
                         */
                        bpf_sk_reuseport_detach(sk);

                        __reuseport_detach_sock(sk, reuse);
                        __reuseport_add_closed_sock(sk, reuse);

                        spin_unlock_bh(&reuseport_lock);
                        return;
                }

                spin_unlock_bh(&reuseport_lock);
        }

        /* Not capable to do migration, detach immediately */
        reuseport_detach_sock(sk);
}
EXPORT_SYMBOL(reuseport_stop_listen_sock);

static struct sock *run_bpf_filter(struct sock_reuseport *reuse, u16 socks,
                                   struct bpf_prog *prog, struct sk_buff *skb,
                                   int hdr_len)
{
        struct sk_buff *nskb = NULL;
        u32 index;

        if (skb_shared(skb)) {
                nskb = skb_clone(skb, GFP_ATOMIC);
                if (!nskb)
                        return NULL;
                skb = nskb;
        }

        /* temporarily advance data past protocol header */
        if (!pskb_pull(skb, hdr_len)) {
                kfree_skb(nskb);
                return NULL;
        }
        index = bpf_prog_run_save_cb(prog, skb);
        __skb_push(skb, hdr_len);

        consume_skb(nskb);

        if (index >= socks)
                return NULL;

        return reuse->socks[index];
}

static struct sock *reuseport_select_sock_by_hash(struct sock_reuseport *reuse,
                                                  u32 hash, u16 num_socks)
{
        struct sock *first_valid_sk = NULL;
        int i, j;

        i = j = reciprocal_scale(hash, num_socks);
        do {
                struct sock *sk = reuse->socks[i];

                if (sk->sk_state != TCP_ESTABLISHED) {
                        /* Paired with WRITE_ONCE() in __reuseport_(get|put)_incoming_cpu(). */
                        if (!READ_ONCE(reuse->incoming_cpu))
                                return sk;

                        /* Paired with WRITE_ONCE() in reuseport_update_incoming_cpu(). */
                        if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
                                return sk;

                        if (!first_valid_sk)
                                first_valid_sk = sk;
                }

                i++;
                if (i >= num_socks)
                        i = 0;
        } while (i != j);

        return first_valid_sk;
}

/**
 *  reuseport_select_sock - Select a socket from an SO_REUSEPORT group.
 *  @sk: First socket in the group.
 *  @hash: When no BPF filter is available, use this hash to select.
 *  @skb: skb to run through BPF filter.
 *  @hdr_len: BPF filter expects skb data pointer at payload data.  If
 *    the skb does not yet point at the payload, this parameter represents
 *    how far the pointer needs to advance to reach the payload.
 *  Returns a socket that should receive the packet (or NULL on error).
 */
struct sock *reuseport_select_sock(struct sock *sk,
                                   u32 hash,
                                   struct sk_buff *skb,
                                   int hdr_len)
{
        struct sock_reuseport *reuse;
        struct bpf_prog *prog;
        struct sock *sk2 = NULL;
        u16 socks;

        rcu_read_lock();
        reuse = rcu_dereference(sk->sk_reuseport_cb);

        /* if memory allocation failed or add call is not yet complete */
        if (!reuse)
                goto out;

        prog = rcu_dereference(reuse->prog);
        socks = READ_ONCE(reuse->num_socks);
        if (likely(socks)) {
                /* paired with smp_wmb() in __reuseport_add_sock() */
                smp_rmb();

                if (!prog || !skb)
                        goto select_by_hash;

                if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT)
                        sk2 = bpf_run_sk_reuseport(reuse, sk, prog, skb, NULL, hash);
                else
                        sk2 = run_bpf_filter(reuse, socks, prog, skb, hdr_len);

select_by_hash:
                /* no bpf or invalid bpf result: fall back to hash usage */
                if (!sk2)
                        sk2 = reuseport_select_sock_by_hash(reuse, hash, socks);
        }

out:
        rcu_read_unlock();
        return sk2;
}
EXPORT_SYMBOL(reuseport_select_sock);

/**
 *  reuseport_migrate_sock - Select a socket from an SO_REUSEPORT group.
 *  @sk: close()ed or shutdown()ed socket in the group.
 *  @migrating_sk: ESTABLISHED/SYN_RECV full socket in the accept queue or
 *    NEW_SYN_RECV request socket during 3WHS.
 *  @skb: skb to run through BPF filter.
 *  Returns a socket (with sk_refcnt +1) that should accept the child socket
 *  (or NULL on error).
 */
struct sock *reuseport_migrate_sock(struct sock *sk,
                                    struct sock *migrating_sk,
                                    struct sk_buff *skb)
{
        struct sock_reuseport *reuse;
        struct sock *nsk = NULL;
        bool allocated = false;
        struct bpf_prog *prog;
        u16 socks;
        u32 hash;

        rcu_read_lock();

        reuse = rcu_dereference(sk->sk_reuseport_cb);
        if (!reuse)
                goto out;

        socks = READ_ONCE(reuse->num_socks);
        if (unlikely(!socks))
                goto failure;

        /* paired with smp_wmb() in __reuseport_add_sock() */
        smp_rmb();

        hash = migrating_sk->sk_hash;
        prog = rcu_dereference(reuse->prog);
        if (!prog || prog->expected_attach_type != BPF_SK_REUSEPORT_SELECT_OR_MIGRATE) {
                if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_migrate_req))
                        goto select_by_hash;
                goto failure;
        }

        if (!skb) {
                skb = alloc_skb(0, GFP_ATOMIC);
                if (!skb)
                        goto failure;
                allocated = true;
        }

        nsk = bpf_run_sk_reuseport(reuse, sk, prog, skb, migrating_sk, hash);

        if (allocated)
                kfree_skb(skb);

select_by_hash:
        if (!nsk)
                nsk = reuseport_select_sock_by_hash(reuse, hash, socks);

        if (IS_ERR_OR_NULL(nsk) || unlikely(!refcount_inc_not_zero(&nsk->sk_refcnt))) {
                nsk = NULL;
                goto failure;
        }

out:
        rcu_read_unlock();
        return nsk;

failure:
        __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQFAILURE);
        goto out;
}
EXPORT_SYMBOL(reuseport_migrate_sock);

int reuseport_attach_prog(struct sock *sk, struct bpf_prog *prog)
{
        struct sock_reuseport *reuse;
        struct bpf_prog *old_prog;

        if (sk_unhashed(sk)) {
                int err;

                if (!sk->sk_reuseport)
                        return -EINVAL;

                err = reuseport_alloc(sk, false);
                if (err)
                        return err;
        } else if (!rcu_access_pointer(sk->sk_reuseport_cb)) {
                /* The socket wasn't bound with SO_REUSEPORT */
                return -EINVAL;
        }

        spin_lock_bh(&reuseport_lock);
        reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
                                          lockdep_is_held(&reuseport_lock));
        old_prog = rcu_dereference_protected(reuse->prog,
                                             lockdep_is_held(&reuseport_lock));
        rcu_assign_pointer(reuse->prog, prog);
        spin_unlock_bh(&reuseport_lock);

        sk_reuseport_prog_free(old_prog);
        return 0;
}
EXPORT_SYMBOL(reuseport_attach_prog);

int reuseport_detach_prog(struct sock *sk)
{
        struct sock_reuseport *reuse;
        struct bpf_prog *old_prog;

        old_prog = NULL;
        spin_lock_bh(&reuseport_lock);
        reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
                                          lockdep_is_held(&reuseport_lock));

        /* reuse must be checked after acquiring the reuseport_lock
         * because reuseport_grow() can detach a closed sk.
         */
        if (!reuse) {
                spin_unlock_bh(&reuseport_lock);
                return sk->sk_reuseport ? -ENOENT : -EINVAL;
        }

        if (sk_unhashed(sk) && reuse->num_closed_socks) {
                spin_unlock_bh(&reuseport_lock);
                return -ENOENT;
        }

        old_prog = rcu_replace_pointer(reuse->prog, old_prog,
                                       lockdep_is_held(&reuseport_lock));
        spin_unlock_bh(&reuseport_lock);

        if (!old_prog)
                return -ENOENT;

        sk_reuseport_prog_free(old_prog);
        return 0;
}
EXPORT_SYMBOL(reuseport_detach_prog);

















































































































































   10 
   10 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
// SPDX-License-Identifier: GPL-2.0+
/*
 * IMA support for appraising module-style appended signatures.
 *
 * Copyright (C) 2019  IBM Corporation
 *
 * Author:
 * Thiago Jung Bauermann <bauerman@linux.ibm.com>
 */

#include <linux/types.h>
#include <linux/module_signature.h>
#include <keys/asymmetric-type.h>
#include <crypto/pkcs7.h>

#include "ima.h"

struct modsig {
        struct pkcs7_message *pkcs7_msg;

        enum hash_algo hash_algo;

        /* This digest will go in the 'd-modsig' field of the IMA template. */
        const u8 *digest;
        u32 digest_size;

        /*
         * This is what will go to the measurement list if the template requires
         * storing the signature.
         */
        int raw_pkcs7_len;
        u8 raw_pkcs7[] __counted_by(raw_pkcs7_len);
};

/*
 * ima_read_modsig - Read modsig from buf.
 *
 * Return: 0 on success, error code otherwise.
 */
int ima_read_modsig(enum ima_hooks func, const void *buf, loff_t buf_len,
                    struct modsig **modsig)
{
        const size_t marker_len = strlen(MODULE_SIG_STRING);
        const struct module_signature *sig;
        struct modsig *hdr;
        size_t sig_len;
        const void *p;
        int rc;

        if (buf_len <= marker_len + sizeof(*sig))
                return -ENOENT;

        p = buf + buf_len - marker_len;
        if (memcmp(p, MODULE_SIG_STRING, marker_len))
                return -ENOENT;

        buf_len -= marker_len;
        sig = (const struct module_signature *)(p - sizeof(*sig));

        rc = mod_check_sig(sig, buf_len, func_tokens[func]);
        if (rc)
                return rc;

        sig_len = be32_to_cpu(sig->sig_len);
        buf_len -= sig_len + sizeof(*sig);

        /* Allocate sig_len additional bytes to hold the raw PKCS#7 data. */
        hdr = kzalloc(struct_size(hdr, raw_pkcs7, sig_len), GFP_KERNEL);
        if (!hdr)
                return -ENOMEM;

        hdr->raw_pkcs7_len = sig_len;
        hdr->pkcs7_msg = pkcs7_parse_message(buf + buf_len, sig_len);
        if (IS_ERR(hdr->pkcs7_msg)) {
                rc = PTR_ERR(hdr->pkcs7_msg);
                kfree(hdr);
                return rc;
        }

        memcpy(hdr->raw_pkcs7, buf + buf_len, sig_len);

        /* We don't know the hash algorithm yet. */
        hdr->hash_algo = HASH_ALGO__LAST;

        *modsig = hdr;

        return 0;
}

/**
 * ima_collect_modsig - Calculate the file hash without the appended signature.
 * @modsig: parsed module signature
 * @buf: data to verify the signature on
 * @size: data size
 *
 * Since the modsig is part of the file contents, the hash used in its signature
 * isn't the same one ordinarily calculated by IMA. Therefore PKCS7 code
 * calculates a separate one for signature verification.
 */
void ima_collect_modsig(struct modsig *modsig, const void *buf, loff_t size)
{
        int rc;

        /*
         * Provide the file contents (minus the appended sig) so that the PKCS7
         * code can calculate the file hash.
         */
        size -= modsig->raw_pkcs7_len + strlen(MODULE_SIG_STRING) +
                sizeof(struct module_signature);
        rc = pkcs7_supply_detached_data(modsig->pkcs7_msg, buf, size);
        if (rc)
                return;

        /* Ask the PKCS7 code to calculate the file hash. */
        rc = pkcs7_get_digest(modsig->pkcs7_msg, &modsig->digest,
                              &modsig->digest_size, &modsig->hash_algo);
}

int ima_modsig_verify(struct key *keyring, const struct modsig *modsig)
{
        return verify_pkcs7_message_sig(NULL, 0, modsig->pkcs7_msg, keyring,
                                        VERIFYING_MODULE_SIGNATURE, NULL, NULL);
}

int ima_get_modsig_digest(const struct modsig *modsig, enum hash_algo *algo,
                          const u8 **digest, u32 *digest_size)
{
        *algo = modsig->hash_algo;
        *digest = modsig->digest;
        *digest_size = modsig->digest_size;

        return 0;
}

int ima_get_raw_modsig(const struct modsig *modsig, const void **data,
                       u32 *data_len)
{
        *data = &modsig->raw_pkcs7;
        *data_len = modsig->raw_pkcs7_len;

        return 0;
}

void ima_free_modsig(struct modsig *modsig)
{
        if (!modsig)
                return;

        pkcs7_free_message(modsig->pkcs7_msg);
        kfree(modsig);
}




















































































































































































































   29 




































   29 








   29 




























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
// SPDX-License-Identifier: GPL-2.0
/*
 * xfrm6_policy.c: based on xfrm4_policy.c
 *
 * Authors:
 *        Mitsuru KANDA @USAGI
 *        Kazunori MIYAZAWA @USAGI
 *        Kunihiro Ishiguro <kunihiro@ipinfusion.com>
 *                IPv6 support
 *        YOSHIFUJI Hideaki
 *                Split up af-specific portion
 *
 */

#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <net/addrconf.h>
#include <net/dst.h>
#include <net/xfrm.h>
#include <net/ip.h>
#include <net/ipv6.h>
#include <net/ip6_route.h>
#include <net/l3mdev.h>

static struct dst_entry *xfrm6_dst_lookup(const struct xfrm_dst_lookup_params *params)
{
        struct flowi6 fl6;
        struct dst_entry *dst;
        int err;

        memset(&fl6, 0, sizeof(fl6));
        fl6.flowi6_l3mdev = l3mdev_master_ifindex_by_index(params->net,
                                                           params->oif);
        fl6.flowi6_mark = params->mark;
        memcpy(&fl6.daddr, params->daddr, sizeof(fl6.daddr));
        if (params->saddr)
                memcpy(&fl6.saddr, params->saddr, sizeof(fl6.saddr));

        fl6.flowi4_proto = params->ipproto;
        fl6.uli = params->uli;

        dst = ip6_route_output(params->net, NULL, &fl6);

        err = dst->error;
        if (dst->error) {
                dst_release(dst);
                dst = ERR_PTR(err);
        }

        return dst;
}

static int xfrm6_get_saddr(xfrm_address_t *saddr,
                           const struct xfrm_dst_lookup_params *params)
{
        struct dst_entry *dst;
        struct net_device *dev;
        struct inet6_dev *idev;

        dst = xfrm6_dst_lookup(params);
        if (IS_ERR(dst))
                return -EHOSTUNREACH;

        idev = ip6_dst_idev(dst);
        if (!idev) {
                dst_release(dst);
                return -EHOSTUNREACH;
        }
        dev = idev->dev;
        ipv6_dev_get_saddr(dev_net(dev), dev, &params->daddr->in6, 0,
                           &saddr->in6);
        dst_release(dst);
        return 0;
}

static int xfrm6_fill_dst(struct xfrm_dst *xdst, struct net_device *dev,
                          const struct flowi *fl)
{
        struct rt6_info *rt = dst_rt6_info(xdst->route);

        xdst->u.dst.dev = dev;
        netdev_hold(dev, &xdst->u.dst.dev_tracker, GFP_ATOMIC);

        xdst->u.rt6.rt6i_idev = in6_dev_get(dev);
        if (!xdst->u.rt6.rt6i_idev) {
                netdev_put(dev, &xdst->u.dst.dev_tracker);
                return -ENODEV;
        }

        /* Sheit... I remember I did this right. Apparently,
         * it was magically lost, so this code needs audit */
        xdst->u.rt6.rt6i_flags = rt->rt6i_flags & (RTF_ANYCAST |
                                                   RTF_LOCAL);
        xdst->route_cookie = rt6_get_cookie(rt);
        xdst->u.rt6.rt6i_gateway = rt->rt6i_gateway;
        xdst->u.rt6.rt6i_dst = rt->rt6i_dst;
        xdst->u.rt6.rt6i_src = rt->rt6i_src;
        rt6_uncached_list_add(&xdst->u.rt6);

        return 0;
}

static void xfrm6_update_pmtu(struct dst_entry *dst, struct sock *sk,
                              struct sk_buff *skb, u32 mtu,
                              bool confirm_neigh)
{
        struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
        struct dst_entry *path = xdst->route;

        path->ops->update_pmtu(path, sk, skb, mtu, confirm_neigh);
}

static void xfrm6_redirect(struct dst_entry *dst, struct sock *sk,
                           struct sk_buff *skb)
{
        struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
        struct dst_entry *path = xdst->route;

        path->ops->redirect(path, sk, skb);
}

static void xfrm6_dst_destroy(struct dst_entry *dst)
{
        struct xfrm_dst *xdst = (struct xfrm_dst *)dst;

        dst_destroy_metrics_generic(dst);
        rt6_uncached_list_del(&xdst->u.rt6);
        if (likely(xdst->u.rt6.rt6i_idev))
                in6_dev_put(xdst->u.rt6.rt6i_idev);
        xfrm_dst_destroy(xdst);
}

static void xfrm6_dst_ifdown(struct dst_entry *dst, struct net_device *dev)
{
        struct xfrm_dst *xdst;

        xdst = (struct xfrm_dst *)dst;
        if (xdst->u.rt6.rt6i_idev->dev == dev) {
                struct inet6_dev *loopback_idev =
                        in6_dev_get(dev_net(dev)->loopback_dev);

                do {
                        in6_dev_put(xdst->u.rt6.rt6i_idev);
                        xdst->u.rt6.rt6i_idev = loopback_idev;
                        in6_dev_hold(loopback_idev);
                        xdst = (struct xfrm_dst *)xfrm_dst_child(&xdst->u.dst);
                } while (xdst->u.dst.xfrm);

                __in6_dev_put(loopback_idev);
        }

        xfrm_dst_ifdown(dst, dev);
}

static struct dst_ops xfrm6_dst_ops_template = {
        .family =                AF_INET6,
        .update_pmtu =                xfrm6_update_pmtu,
        .redirect =                xfrm6_redirect,
        .cow_metrics =                dst_cow_metrics_generic,
        .destroy =                xfrm6_dst_destroy,
        .ifdown =                xfrm6_dst_ifdown,
        .local_out =                __ip6_local_out,
        .gc_thresh =                32768,
};

static const struct xfrm_policy_afinfo xfrm6_policy_afinfo = {
        .dst_ops =                &xfrm6_dst_ops_template,
        .dst_lookup =                xfrm6_dst_lookup,
        .get_saddr =                xfrm6_get_saddr,
        .fill_dst =                xfrm6_fill_dst,
        .blackhole_route =        ip6_blackhole_route,
};

static int __init xfrm6_policy_init(void)
{
        return xfrm_policy_register_afinfo(&xfrm6_policy_afinfo, AF_INET6);
}

static void xfrm6_policy_fini(void)
{
        xfrm_policy_unregister_afinfo(&xfrm6_policy_afinfo);
}

#ifdef CONFIG_SYSCTL
static struct ctl_table xfrm6_policy_table[] = {
        {
                .procname       = "xfrm6_gc_thresh",
                .data                = &init_net.xfrm.xfrm6_dst_ops.gc_thresh,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler   = proc_dointvec,
        },
};

static int __net_init xfrm6_net_sysctl_init(struct net *net)
{
        struct ctl_table *table;
        struct ctl_table_header *hdr;

        table = xfrm6_policy_table;
        if (!net_eq(net, &init_net)) {
                table = kmemdup(table, sizeof(xfrm6_policy_table), GFP_KERNEL);
                if (!table)
                        goto err_alloc;

                table[0].data = &net->xfrm.xfrm6_dst_ops.gc_thresh;
        }

        hdr = register_net_sysctl_sz(net, "net/ipv6", table,
                                     ARRAY_SIZE(xfrm6_policy_table));
        if (!hdr)
                goto err_reg;

        net->ipv6.sysctl.xfrm6_hdr = hdr;
        return 0;

err_reg:
        if (!net_eq(net, &init_net))
                kfree(table);
err_alloc:
        return -ENOMEM;
}

static void __net_exit xfrm6_net_sysctl_exit(struct net *net)
{
        const struct ctl_table *table;

        if (!net->ipv6.sysctl.xfrm6_hdr)
                return;

        table = net->ipv6.sysctl.xfrm6_hdr->ctl_table_arg;
        unregister_net_sysctl_table(net->ipv6.sysctl.xfrm6_hdr);
        if (!net_eq(net, &init_net))
                kfree(table);
}
#else /* CONFIG_SYSCTL */
static inline int xfrm6_net_sysctl_init(struct net *net)
{
        return 0;
}

static inline void xfrm6_net_sysctl_exit(struct net *net)
{
}
#endif

static int __net_init xfrm6_net_init(struct net *net)
{
        int ret;

        memcpy(&net->xfrm.xfrm6_dst_ops, &xfrm6_dst_ops_template,
               sizeof(xfrm6_dst_ops_template));
        ret = dst_entries_init(&net->xfrm.xfrm6_dst_ops);
        if (ret)
                return ret;

        ret = xfrm6_net_sysctl_init(net);
        if (ret)
                dst_entries_destroy(&net->xfrm.xfrm6_dst_ops);

        return ret;
}

static void __net_exit xfrm6_net_exit(struct net *net)
{
        xfrm6_net_sysctl_exit(net);
        dst_entries_destroy(&net->xfrm.xfrm6_dst_ops);
}

static struct pernet_operations xfrm6_net_ops = {
        .init        = xfrm6_net_init,
        .exit        = xfrm6_net_exit,
};

int __init xfrm6_init(void)
{
        int ret;

        ret = xfrm6_policy_init();
        if (ret)
                goto out;
        ret = xfrm6_state_init();
        if (ret)
                goto out_policy;

        ret = xfrm6_protocol_init();
        if (ret)
                goto out_state;

        ret = register_pernet_subsys(&xfrm6_net_ops);
        if (ret)
                goto out_protocol;

        ret = xfrm_nat_keepalive_init(AF_INET6);
        if (ret)
                goto out_nat_keepalive;
out:
        return ret;
out_nat_keepalive:
        unregister_pernet_subsys(&xfrm6_net_ops);
out_protocol:
        xfrm6_protocol_fini();
out_state:
        xfrm6_state_fini();
out_policy:
        xfrm6_policy_fini();
        goto out;
}

void xfrm6_fini(void)
{
        xfrm_nat_keepalive_fini(AF_INET6);
        unregister_pernet_subsys(&xfrm6_net_ops);
        xfrm6_protocol_fini();
        xfrm6_policy_fini();
        xfrm6_state_fini();
}




















































































































   29 
   29 




























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   29 


   29 


























   29 









































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * IPVS         An implementation of the IP virtual server support for the
 *              LINUX operating system.  IPVS is now implemented as a module
 *              over the Netfilter framework. IPVS can be used to build a
 *              high-performance and highly available server based on a
 *              cluster of servers.
 *
 * Authors:     Wensong Zhang <wensong@linuxvirtualserver.org>
 *              Peter Kese <peter.kese@ijs.si>
 *              Julian Anastasov <ja@ssi.bg>
 *
 * The IPVS code for kernel 2.2 was done by Wensong Zhang and Peter Kese,
 * with changes/fixes from Julian Anastasov, Lars Marowsky-Bree, Horms
 * and others.
 *
 * Changes:
 *        Paul `Rusty' Russell                properly handle non-linear skbs
 *        Harald Welte                        don't use nfcache
 */

#define KMSG_COMPONENT "IPVS"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/sctp.h>
#include <linux/icmp.h>
#include <linux/slab.h>

#include <net/ip.h>
#include <net/tcp.h>
#include <net/udp.h>
#include <net/icmp.h>                   /* for icmp_send */
#include <net/gue.h>
#include <net/gre.h>
#include <net/route.h>
#include <net/ip6_checksum.h>
#include <net/netns/generic.h>                /* net_generic() */

#include <linux/netfilter.h>
#include <linux/netfilter_ipv4.h>

#ifdef CONFIG_IP_VS_IPV6
#include <net/ipv6.h>
#include <linux/netfilter_ipv6.h>
#include <net/ip6_route.h>
#endif

#include <net/ip_vs.h>
#include <linux/indirect_call_wrapper.h>


EXPORT_SYMBOL(register_ip_vs_scheduler);
EXPORT_SYMBOL(unregister_ip_vs_scheduler);
EXPORT_SYMBOL(ip_vs_proto_name);
EXPORT_SYMBOL(ip_vs_conn_new);
EXPORT_SYMBOL(ip_vs_conn_in_get);
EXPORT_SYMBOL(ip_vs_conn_out_get);
#ifdef CONFIG_IP_VS_PROTO_TCP
EXPORT_SYMBOL(ip_vs_tcp_conn_listen);
#endif
EXPORT_SYMBOL(ip_vs_conn_put);
#ifdef CONFIG_IP_VS_DEBUG
EXPORT_SYMBOL(ip_vs_get_debug_level);
#endif
EXPORT_SYMBOL(ip_vs_new_conn_out);

#if defined(CONFIG_IP_VS_PROTO_TCP) && defined(CONFIG_IP_VS_PROTO_UDP)
#define SNAT_CALL(f, ...) \
        INDIRECT_CALL_2(f, tcp_snat_handler, udp_snat_handler, __VA_ARGS__)
#elif defined(CONFIG_IP_VS_PROTO_TCP)
#define SNAT_CALL(f, ...) INDIRECT_CALL_1(f, tcp_snat_handler, __VA_ARGS__)
#elif defined(CONFIG_IP_VS_PROTO_UDP)
#define SNAT_CALL(f, ...) INDIRECT_CALL_1(f, udp_snat_handler, __VA_ARGS__)
#else
#define SNAT_CALL(f, ...) f(__VA_ARGS__)
#endif

static unsigned int ip_vs_net_id __read_mostly;
/* netns cnt used for uniqueness */
static atomic_t ipvs_netns_cnt = ATOMIC_INIT(0);

/* ID used in ICMP lookups */
#define icmp_id(icmph)          (((icmph)->un).echo.id)
#define icmpv6_id(icmph)        (icmph->icmp6_dataun.u_echo.identifier)

const char *ip_vs_proto_name(unsigned int proto)
{
        static char buf[20];

        switch (proto) {
        case IPPROTO_IP:
                return "IP";
        case IPPROTO_UDP:
                return "UDP";
        case IPPROTO_TCP:
                return "TCP";
        case IPPROTO_SCTP:
                return "SCTP";
        case IPPROTO_ICMP:
                return "ICMP";
#ifdef CONFIG_IP_VS_IPV6
        case IPPROTO_ICMPV6:
                return "ICMPv6";
#endif
        default:
                sprintf(buf, "IP_%u", proto);
                return buf;
        }
}

void ip_vs_init_hash_table(struct list_head *table, int rows)
{
        while (--rows >= 0)
                INIT_LIST_HEAD(&table[rows]);
}

static inline void
ip_vs_in_stats(struct ip_vs_conn *cp, struct sk_buff *skb)
{
        struct ip_vs_dest *dest = cp->dest;
        struct netns_ipvs *ipvs = cp->ipvs;

        if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {
                struct ip_vs_cpu_stats *s;
                struct ip_vs_service *svc;

                local_bh_disable();

                s = this_cpu_ptr(dest->stats.cpustats);
                u64_stats_update_begin(&s->syncp);
                u64_stats_inc(&s->cnt.inpkts);
                u64_stats_add(&s->cnt.inbytes, skb->len);
                u64_stats_update_end(&s->syncp);

                svc = rcu_dereference(dest->svc);
                s = this_cpu_ptr(svc->stats.cpustats);
                u64_stats_update_begin(&s->syncp);
                u64_stats_inc(&s->cnt.inpkts);
                u64_stats_add(&s->cnt.inbytes, skb->len);
                u64_stats_update_end(&s->syncp);

                s = this_cpu_ptr(ipvs->tot_stats->s.cpustats);
                u64_stats_update_begin(&s->syncp);
                u64_stats_inc(&s->cnt.inpkts);
                u64_stats_add(&s->cnt.inbytes, skb->len);
                u64_stats_update_end(&s->syncp);

                local_bh_enable();
        }
}


static inline void
ip_vs_out_stats(struct ip_vs_conn *cp, struct sk_buff *skb)
{
        struct ip_vs_dest *dest = cp->dest;
        struct netns_ipvs *ipvs = cp->ipvs;

        if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {
                struct ip_vs_cpu_stats *s;
                struct ip_vs_service *svc;

                local_bh_disable();

                s = this_cpu_ptr(dest->stats.cpustats);
                u64_stats_update_begin(&s->syncp);
                u64_stats_inc(&s->cnt.outpkts);
                u64_stats_add(&s->cnt.outbytes, skb->len);
                u64_stats_update_end(&s->syncp);

                svc = rcu_dereference(dest->svc);
                s = this_cpu_ptr(svc->stats.cpustats);
                u64_stats_update_begin(&s->syncp);
                u64_stats_inc(&s->cnt.outpkts);
                u64_stats_add(&s->cnt.outbytes, skb->len);
                u64_stats_update_end(&s->syncp);

                s = this_cpu_ptr(ipvs->tot_stats->s.cpustats);
                u64_stats_update_begin(&s->syncp);
                u64_stats_inc(&s->cnt.outpkts);
                u64_stats_add(&s->cnt.outbytes, skb->len);
                u64_stats_update_end(&s->syncp);

                local_bh_enable();
        }
}


static inline void
ip_vs_conn_stats(struct ip_vs_conn *cp, struct ip_vs_service *svc)
{
        struct netns_ipvs *ipvs = svc->ipvs;
        struct ip_vs_cpu_stats *s;

        local_bh_disable();

        s = this_cpu_ptr(cp->dest->stats.cpustats);
        u64_stats_update_begin(&s->syncp);
        u64_stats_inc(&s->cnt.conns);
        u64_stats_update_end(&s->syncp);

        s = this_cpu_ptr(svc->stats.cpustats);
        u64_stats_update_begin(&s->syncp);
        u64_stats_inc(&s->cnt.conns);
        u64_stats_update_end(&s->syncp);

        s = this_cpu_ptr(ipvs->tot_stats->s.cpustats);
        u64_stats_update_begin(&s->syncp);
        u64_stats_inc(&s->cnt.conns);
        u64_stats_update_end(&s->syncp);

        local_bh_enable();
}


static inline void
ip_vs_set_state(struct ip_vs_conn *cp, int direction,
                const struct sk_buff *skb,
                struct ip_vs_proto_data *pd)
{
        if (likely(pd->pp->state_transition))
                pd->pp->state_transition(cp, direction, skb, pd);
}

static inline int
ip_vs_conn_fill_param_persist(const struct ip_vs_service *svc,
                              struct sk_buff *skb, int protocol,
                              const union nf_inet_addr *caddr, __be16 cport,
                              const union nf_inet_addr *vaddr, __be16 vport,
                              struct ip_vs_conn_param *p)
{
        ip_vs_conn_fill_param(svc->ipvs, svc->af, protocol, caddr, cport, vaddr,
                              vport, p);
        p->pe = rcu_dereference(svc->pe);
        if (p->pe && p->pe->fill_param)
                return p->pe->fill_param(p, skb);

        return 0;
}

/*
 *  IPVS persistent scheduling function
 *  It creates a connection entry according to its template if exists,
 *  or selects a server and creates a connection entry plus a template.
 *  Locking: we are svc user (svc->refcnt), so we hold all dests too
 *  Protocols supported: TCP, UDP
 */
static struct ip_vs_conn *
ip_vs_sched_persist(struct ip_vs_service *svc,
                    struct sk_buff *skb, __be16 src_port, __be16 dst_port,
                    int *ignored, struct ip_vs_iphdr *iph)
{
        struct ip_vs_conn *cp = NULL;
        struct ip_vs_dest *dest;
        struct ip_vs_conn *ct;
        __be16 dport = 0;                /* destination port to forward */
        unsigned int flags;
        struct ip_vs_conn_param param;
        const union nf_inet_addr fwmark = { .ip = htonl(svc->fwmark) };
        union nf_inet_addr snet;        /* source network of the client,
                                           after masking */
        const union nf_inet_addr *src_addr, *dst_addr;

        if (likely(!ip_vs_iph_inverse(iph))) {
                src_addr = &iph->saddr;
                dst_addr = &iph->daddr;
        } else {
                src_addr = &iph->daddr;
                dst_addr = &iph->saddr;
        }


        /* Mask saddr with the netmask to adjust template granularity */
#ifdef CONFIG_IP_VS_IPV6
        if (svc->af == AF_INET6)
                ipv6_addr_prefix(&snet.in6, &src_addr->in6,
                                 (__force __u32) svc->netmask);
        else
#endif
                snet.ip = src_addr->ip & svc->netmask;

        IP_VS_DBG_BUF(6, "p-schedule: src %s:%u dest %s:%u "
                      "mnet %s\n",
                      IP_VS_DBG_ADDR(svc->af, src_addr), ntohs(src_port),
                      IP_VS_DBG_ADDR(svc->af, dst_addr), ntohs(dst_port),
                      IP_VS_DBG_ADDR(svc->af, &snet));

        /*
         * As far as we know, FTP is a very complicated network protocol, and
         * it uses control connection and data connections. For active FTP,
         * FTP server initialize data connection to the client, its source port
         * is often 20. For passive FTP, FTP server tells the clients the port
         * that it passively listens to,  and the client issues the data
         * connection. In the tunneling or direct routing mode, the load
         * balancer is on the client-to-server half of connection, the port
         * number is unknown to the load balancer. So, a conn template like
         * <caddr, 0, vaddr, 0, daddr, 0> is created for persistent FTP
         * service, and a template like <caddr, 0, vaddr, vport, daddr, dport>
         * is created for other persistent services.
         */
        {
                int protocol = iph->protocol;
                const union nf_inet_addr *vaddr = dst_addr;
                __be16 vport = 0;

                if (dst_port == svc->port) {
                        /* non-FTP template:
                         * <protocol, caddr, 0, vaddr, vport, daddr, dport>
                         * FTP template:
                         * <protocol, caddr, 0, vaddr, 0, daddr, 0>
                         */
                        if (svc->port != FTPPORT)
                                vport = dst_port;
                } else {
                        /* Note: persistent fwmark-based services and
                         * persistent port zero service are handled here.
                         * fwmark template:
                         * <IPPROTO_IP,caddr,0,fwmark,0,daddr,0>
                         * port zero template:
                         * <protocol,caddr,0,vaddr,0,daddr,0>
                         */
                        if (svc->fwmark) {
                                protocol = IPPROTO_IP;
                                vaddr = &fwmark;
                        }
                }
                /* return *ignored = -1 so NF_DROP can be used */
                if (ip_vs_conn_fill_param_persist(svc, skb, protocol, &snet, 0,
                                                  vaddr, vport, &param) < 0) {
                        *ignored = -1;
                        return NULL;
                }
        }

        /* Check if a template already exists */
        ct = ip_vs_ct_in_get(&param);
        if (!ct || !ip_vs_check_template(ct, NULL)) {
                struct ip_vs_scheduler *sched;

                /*
                 * No template found or the dest of the connection
                 * template is not available.
                 * return *ignored=0 i.e. ICMP and NF_DROP
                 */
                sched = rcu_dereference(svc->scheduler);
                if (sched) {
                        /* read svc->sched_data after svc->scheduler */
                        smp_rmb();
                        dest = sched->schedule(svc, skb, iph);
                } else {
                        dest = NULL;
                }
                if (!dest) {
                        IP_VS_DBG(1, "p-schedule: no dest found.\n");
                        kfree(param.pe_data);
                        *ignored = 0;
                        return NULL;
                }

                if (dst_port == svc->port && svc->port != FTPPORT)
                        dport = dest->port;

                /* Create a template
                 * This adds param.pe_data to the template,
                 * and thus param.pe_data will be destroyed
                 * when the template expires */
                ct = ip_vs_conn_new(&param, dest->af, &dest->addr, dport,
                                    IP_VS_CONN_F_TEMPLATE, dest, skb->mark);
                if (ct == NULL) {
                        kfree(param.pe_data);
                        *ignored = -1;
                        return NULL;
                }

                ct->timeout = svc->timeout;
        } else {
                /* set destination with the found template */
                dest = ct->dest;
                kfree(param.pe_data);
        }

        dport = dst_port;
        if (dport == svc->port && dest->port)
                dport = dest->port;

        flags = (svc->flags & IP_VS_SVC_F_ONEPACKET
                 && iph->protocol == IPPROTO_UDP) ?
                IP_VS_CONN_F_ONE_PACKET : 0;

        /*
         *    Create a new connection according to the template
         */
        ip_vs_conn_fill_param(svc->ipvs, svc->af, iph->protocol, src_addr,
                              src_port, dst_addr, dst_port, &param);

        cp = ip_vs_conn_new(&param, dest->af, &dest->addr, dport, flags, dest,
                            skb->mark);
        if (cp == NULL) {
                ip_vs_conn_put(ct);
                *ignored = -1;
                return NULL;
        }

        /*
         *    Add its control
         */
        ip_vs_control_add(cp, ct);
        ip_vs_conn_put(ct);

        ip_vs_conn_stats(cp, svc);
        return cp;
}


/*
 *  IPVS main scheduling function
 *  It selects a server according to the virtual service, and
 *  creates a connection entry.
 *  Protocols supported: TCP, UDP
 *
 *  Usage of *ignored
 *
 * 1 :   protocol tried to schedule (eg. on SYN), found svc but the
 *       svc/scheduler decides that this packet should be accepted with
 *       NF_ACCEPT because it must not be scheduled.
 *
 * 0 :   scheduler can not find destination, so try bypass or
 *       return ICMP and then NF_DROP (ip_vs_leave).
 *
 * -1 :  scheduler tried to schedule but fatal error occurred, eg.
 *       ip_vs_conn_new failure (ENOMEM) or ip_vs_sip_fill_param
 *       failure such as missing Call-ID, ENOMEM on skb_linearize
 *       or pe_data. In this case we should return NF_DROP without
 *       any attempts to send ICMP with ip_vs_leave.
 */
struct ip_vs_conn *
ip_vs_schedule(struct ip_vs_service *svc, struct sk_buff *skb,
               struct ip_vs_proto_data *pd, int *ignored,
               struct ip_vs_iphdr *iph)
{
        struct ip_vs_protocol *pp = pd->pp;
        struct ip_vs_conn *cp = NULL;
        struct ip_vs_scheduler *sched;
        struct ip_vs_dest *dest;
        __be16 _ports[2], *pptr, cport, vport;
        const void *caddr, *vaddr;
        unsigned int flags;

        *ignored = 1;
        /*
         * IPv6 frags, only the first hit here.
         */
        pptr = frag_safe_skb_hp(skb, iph->len, sizeof(_ports), _ports);
        if (pptr == NULL)
                return NULL;

        if (likely(!ip_vs_iph_inverse(iph))) {
                cport = pptr[0];
                caddr = &iph->saddr;
                vport = pptr[1];
                vaddr = &iph->daddr;
        } else {
                cport = pptr[1];
                caddr = &iph->daddr;
                vport = pptr[0];
                vaddr = &iph->saddr;
        }

        /*
         * FTPDATA needs this check when using local real server.
         * Never schedule Active FTPDATA connections from real server.
         * For LVS-NAT they must be already created. For other methods
         * with persistence the connection is created on SYN+ACK.
         */
        if (cport == FTPDATA) {
                IP_VS_DBG_PKT(12, svc->af, pp, skb, iph->off,
                              "Not scheduling FTPDATA");
                return NULL;
        }

        /*
         *    Do not schedule replies from local real server.
         */
        if ((!skb->dev || skb->dev->flags & IFF_LOOPBACK)) {
                iph->hdr_flags ^= IP_VS_HDR_INVERSE;
                cp = INDIRECT_CALL_1(pp->conn_in_get,
                                     ip_vs_conn_in_get_proto, svc->ipvs,
                                     svc->af, skb, iph);
                iph->hdr_flags ^= IP_VS_HDR_INVERSE;

                if (cp) {
                        IP_VS_DBG_PKT(12, svc->af, pp, skb, iph->off,
                                      "Not scheduling reply for existing"
                                      " connection");
                        __ip_vs_conn_put(cp);
                        return NULL;
                }
        }

        /*
         *    Persistent service
         */
        if (svc->flags & IP_VS_SVC_F_PERSISTENT)
                return ip_vs_sched_persist(svc, skb, cport, vport, ignored,
                                           iph);

        *ignored = 0;

        /*
         *    Non-persistent service
         */
        if (!svc->fwmark && vport != svc->port) {
                if (!svc->port)
                        pr_err("Schedule: port zero only supported "
                               "in persistent services, "
                               "check your ipvs configuration\n");
                return NULL;
        }

        sched = rcu_dereference(svc->scheduler);
        if (sched) {
                /* read svc->sched_data after svc->scheduler */
                smp_rmb();
                dest = sched->schedule(svc, skb, iph);
        } else {
                dest = NULL;
        }
        if (dest == NULL) {
                IP_VS_DBG(1, "Schedule: no dest found.\n");
                return NULL;
        }

        flags = (svc->flags & IP_VS_SVC_F_ONEPACKET
                 && iph->protocol == IPPROTO_UDP) ?
                IP_VS_CONN_F_ONE_PACKET : 0;

        /*
         *    Create a connection entry.
         */
        {
                struct ip_vs_conn_param p;

                ip_vs_conn_fill_param(svc->ipvs, svc->af, iph->protocol,
                                      caddr, cport, vaddr, vport, &p);
                cp = ip_vs_conn_new(&p, dest->af, &dest->addr,
                                    dest->port ? dest->port : vport,
                                    flags, dest, skb->mark);
                if (!cp) {
                        *ignored = -1;
                        return NULL;
                }
        }

        IP_VS_DBG_BUF(6, "Schedule fwd:%c c:%s:%u v:%s:%u "
                      "d:%s:%u conn->flags:%X conn->refcnt:%d\n",
                      ip_vs_fwd_tag(cp),
                      IP_VS_DBG_ADDR(cp->af, &cp->caddr), ntohs(cp->cport),
                      IP_VS_DBG_ADDR(cp->af, &cp->vaddr), ntohs(cp->vport),
                      IP_VS_DBG_ADDR(cp->daf, &cp->daddr), ntohs(cp->dport),
                      cp->flags, refcount_read(&cp->refcnt));

        ip_vs_conn_stats(cp, svc);
        return cp;
}

static inline int ip_vs_addr_is_unicast(struct net *net, int af,
                                        union nf_inet_addr *addr)
{
#ifdef CONFIG_IP_VS_IPV6
        if (af == AF_INET6)
                return ipv6_addr_type(&addr->in6) & IPV6_ADDR_UNICAST;
#endif
        return (inet_addr_type(net, addr->ip) == RTN_UNICAST);
}

/*
 *  Pass or drop the packet.
 *  Called by ip_vs_in, when the virtual service is available but
 *  no destination is available for a new connection.
 */
int ip_vs_leave(struct ip_vs_service *svc, struct sk_buff *skb,
                struct ip_vs_proto_data *pd, struct ip_vs_iphdr *iph)
{
        __be16 _ports[2], *pptr, dport;
        struct netns_ipvs *ipvs = svc->ipvs;
        struct net *net = ipvs->net;

        pptr = frag_safe_skb_hp(skb, iph->len, sizeof(_ports), _ports);
        if (!pptr)
                return NF_DROP;
        dport = likely(!ip_vs_iph_inverse(iph)) ? pptr[1] : pptr[0];

        /* if it is fwmark-based service, the cache_bypass sysctl is up
           and the destination is a non-local unicast, then create
           a cache_bypass connection entry */
        if (sysctl_cache_bypass(ipvs) && svc->fwmark &&
            !(iph->hdr_flags & (IP_VS_HDR_INVERSE | IP_VS_HDR_ICMP)) &&
            ip_vs_addr_is_unicast(net, svc->af, &iph->daddr)) {
                int ret;
                struct ip_vs_conn *cp;
                unsigned int flags = (svc->flags & IP_VS_SVC_F_ONEPACKET &&
                                      iph->protocol == IPPROTO_UDP) ?
                                      IP_VS_CONN_F_ONE_PACKET : 0;
                union nf_inet_addr daddr = { .all = { 0, 0, 0, 0 } };

                /* create a new connection entry */
                IP_VS_DBG(6, "%s(): create a cache_bypass entry\n", __func__);
                {
                        struct ip_vs_conn_param p;
                        ip_vs_conn_fill_param(svc->ipvs, svc->af, iph->protocol,
                                              &iph->saddr, pptr[0],
                                              &iph->daddr, pptr[1], &p);
                        cp = ip_vs_conn_new(&p, svc->af, &daddr, 0,
                                            IP_VS_CONN_F_BYPASS | flags,
                                            NULL, skb->mark);
                        if (!cp)
                                return NF_DROP;
                }

                /* statistics */
                ip_vs_in_stats(cp, skb);

                /* set state */
                ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pd);

                /* transmit the first SYN packet */
                ret = cp->packet_xmit(skb, cp, pd->pp, iph);
                /* do not touch skb anymore */

                if ((cp->flags & IP_VS_CONN_F_ONE_PACKET) && cp->control)
                        atomic_inc(&cp->control->in_pkts);
                else
                        atomic_inc(&cp->in_pkts);
                ip_vs_conn_put(cp);
                return ret;
        }

        /*
         * When the virtual ftp service is presented, packets destined
         * for other services on the VIP may get here (except services
         * listed in the ipvs table), pass the packets, because it is
         * not ipvs job to decide to drop the packets.
         */
        if (svc->port == FTPPORT && dport != FTPPORT)
                return NF_ACCEPT;

        if (unlikely(ip_vs_iph_icmp(iph)))
                return NF_DROP;

        /*
         * Notify the client that the destination is unreachable, and
         * release the socket buffer.
         * Since it is in IP layer, the TCP socket is not actually
         * created, the TCP RST packet cannot be sent, instead that
         * ICMP_PORT_UNREACH is sent here no matter it is TCP/UDP. --WZ
         */
#ifdef CONFIG_IP_VS_IPV6
        if (svc->af == AF_INET6) {
                if (!skb->dev)
                        skb->dev = net->loopback_dev;
                icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_PORT_UNREACH, 0);
        } else
#endif
                icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);

        return NF_DROP;
}

#ifdef CONFIG_SYSCTL

static int sysctl_snat_reroute(struct netns_ipvs *ipvs)
{
        return ipvs->sysctl_snat_reroute;
}

static int sysctl_nat_icmp_send(struct netns_ipvs *ipvs)
{
        return ipvs->sysctl_nat_icmp_send;
}

#else

static int sysctl_snat_reroute(struct netns_ipvs *ipvs) { return 0; }
static int sysctl_nat_icmp_send(struct netns_ipvs *ipvs) { return 0; }

#endif

__sum16 ip_vs_checksum_complete(struct sk_buff *skb, int offset)
{
        return csum_fold(skb_checksum(skb, offset, skb->len - offset, 0));
}

static inline enum ip_defrag_users ip_vs_defrag_user(unsigned int hooknum)
{
        if (NF_INET_LOCAL_IN == hooknum)
                return IP_DEFRAG_VS_IN;
        if (NF_INET_FORWARD == hooknum)
                return IP_DEFRAG_VS_FWD;
        return IP_DEFRAG_VS_OUT;
}

static inline int ip_vs_gather_frags(struct netns_ipvs *ipvs,
                                     struct sk_buff *skb, u_int32_t user)
{
        int err;

        local_bh_disable();
        err = ip_defrag(ipvs->net, skb, user);
        local_bh_enable();
        if (!err)
                ip_send_check(ip_hdr(skb));

        return err;
}

static int ip_vs_route_me_harder(struct netns_ipvs *ipvs, int af,
                                 struct sk_buff *skb, unsigned int hooknum)
{
        if (!sysctl_snat_reroute(ipvs))
                return 0;
        /* Reroute replies only to remote clients (FORWARD and LOCAL_OUT) */
        if (NF_INET_LOCAL_IN == hooknum)
                return 0;
#ifdef CONFIG_IP_VS_IPV6
        if (af == AF_INET6) {
                struct dst_entry *dst = skb_dst(skb);

                if (dst->dev && !(dst->dev->flags & IFF_LOOPBACK) &&
                    ip6_route_me_harder(ipvs->net, skb->sk, skb) != 0)
                        return 1;
        } else
#endif
                if (!(skb_rtable(skb)->rt_flags & RTCF_LOCAL) &&
                    ip_route_me_harder(ipvs->net, skb->sk, skb, RTN_LOCAL) != 0)
                        return 1;

        return 0;
}

/*
 * Packet has been made sufficiently writable in caller
 * - inout: 1=in->out, 0=out->in
 */
void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp,
                    struct ip_vs_conn *cp, int inout)
{
        struct iphdr *iph         = ip_hdr(skb);
        unsigned int icmp_offset = iph->ihl*4;
        struct icmphdr *icmph         = (struct icmphdr *)(skb_network_header(skb) +
                                                      icmp_offset);
        struct iphdr *ciph         = (struct iphdr *)(icmph + 1);

        if (inout) {
                iph->saddr = cp->vaddr.ip;
                ip_send_check(iph);
                ciph->daddr = cp->vaddr.ip;
                ip_send_check(ciph);
        } else {
                iph->daddr = cp->daddr.ip;
                ip_send_check(iph);
                ciph->saddr = cp->daddr.ip;
                ip_send_check(ciph);
        }

        /* the TCP/UDP/SCTP port */
        if (IPPROTO_TCP == ciph->protocol || IPPROTO_UDP == ciph->protocol ||
            IPPROTO_SCTP == ciph->protocol) {
                __be16 *ports = (void *)ciph + ciph->ihl*4;

                if (inout)
                        ports[1] = cp->vport;
                else
                        ports[0] = cp->dport;
        }

        /* And finally the ICMP checksum */
        icmph->checksum = 0;
        icmph->checksum = ip_vs_checksum_complete(skb, icmp_offset);
        skb->ip_summed = CHECKSUM_UNNECESSARY;

        if (inout)
                IP_VS_DBG_PKT(11, AF_INET, pp, skb, (void *)ciph - (void *)iph,
                        "Forwarding altered outgoing ICMP");
        else
                IP_VS_DBG_PKT(11, AF_INET, pp, skb, (void *)ciph - (void *)iph,
                        "Forwarding altered incoming ICMP");
}

#ifdef CONFIG_IP_VS_IPV6
void ip_vs_nat_icmp_v6(struct sk_buff *skb, struct ip_vs_protocol *pp,
                    struct ip_vs_conn *cp, int inout)
{
        struct ipv6hdr *iph         = ipv6_hdr(skb);
        unsigned int icmp_offset = 0;
        unsigned int offs         = 0; /* header offset*/
        int protocol;
        struct icmp6hdr *icmph;
        struct ipv6hdr *ciph;
        unsigned short fragoffs;

        ipv6_find_hdr(skb, &icmp_offset, IPPROTO_ICMPV6, &fragoffs, NULL);
        icmph = (struct icmp6hdr *)(skb_network_header(skb) + icmp_offset);
        offs = icmp_offset + sizeof(struct icmp6hdr);
        ciph = (struct ipv6hdr *)(skb_network_header(skb) + offs);

        protocol = ipv6_find_hdr(skb, &offs, -1, &fragoffs, NULL);

        if (inout) {
                iph->saddr = cp->vaddr.in6;
                ciph->daddr = cp->vaddr.in6;
        } else {
                iph->daddr = cp->daddr.in6;
                ciph->saddr = cp->daddr.in6;
        }

        /* the TCP/UDP/SCTP port */
        if (!fragoffs && (IPPROTO_TCP == protocol || IPPROTO_UDP == protocol ||
                          IPPROTO_SCTP == protocol)) {
                __be16 *ports = (void *)(skb_network_header(skb) + offs);

                IP_VS_DBG(11, "%s() changed port %d to %d\n", __func__,
                              ntohs(inout ? ports[1] : ports[0]),
                              ntohs(inout ? cp->vport : cp->dport));
                if (inout)
                        ports[1] = cp->vport;
                else
                        ports[0] = cp->dport;
        }

        /* And finally the ICMP checksum */
        icmph->icmp6_cksum = ~csum_ipv6_magic(&iph->saddr, &iph->daddr,
                                              skb->len - icmp_offset,
                                              IPPROTO_ICMPV6, 0);
        skb->csum_start = skb_network_header(skb) - skb->head + icmp_offset;
        skb->csum_offset = offsetof(struct icmp6hdr, icmp6_cksum);
        skb->ip_summed = CHECKSUM_PARTIAL;

        if (inout)
                IP_VS_DBG_PKT(11, AF_INET6, pp, skb,
                              (void *)ciph - (void *)iph,
                              "Forwarding altered outgoing ICMPv6");
        else
                IP_VS_DBG_PKT(11, AF_INET6, pp, skb,
                              (void *)ciph - (void *)iph,
                              "Forwarding altered incoming ICMPv6");
}
#endif

/* Handle relevant response ICMP messages - forward to the right
 * destination host.
 */
static int handle_response_icmp(int af, struct sk_buff *skb,
                                union nf_inet_addr *snet,
                                __u8 protocol, struct ip_vs_conn *cp,
                                struct ip_vs_protocol *pp,
                                unsigned int offset, unsigned int ihl,
                                unsigned int hooknum)
{
        unsigned int verdict = NF_DROP;

        if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ)
                goto after_nat;

        /* Ensure the checksum is correct */
        if (!skb_csum_unnecessary(skb) && ip_vs_checksum_complete(skb, ihl)) {
                /* Failed checksum! */
                IP_VS_DBG_BUF(1, "Forward ICMP: failed checksum from %s!\n",
                              IP_VS_DBG_ADDR(af, snet));
                goto out;
        }

        if (IPPROTO_TCP == protocol || IPPROTO_UDP == protocol ||
            IPPROTO_SCTP == protocol)
                offset += 2 * sizeof(__u16);
        if (skb_ensure_writable(skb, offset))
                goto out;

#ifdef CONFIG_IP_VS_IPV6
        if (af == AF_INET6)
                ip_vs_nat_icmp_v6(skb, pp, cp, 1);
        else
#endif
                ip_vs_nat_icmp(skb, pp, cp, 1);

        if (ip_vs_route_me_harder(cp->ipvs, af, skb, hooknum))
                goto out;

after_nat:
        /* do the statistics and put it back */
        ip_vs_out_stats(cp, skb);

        skb->ipvs_property = 1;
        if (!(cp->flags & IP_VS_CONN_F_NFCT))
                ip_vs_notrack(skb);
        else
                ip_vs_update_conntrack(skb, cp, 0);
        verdict = NF_ACCEPT;

out:
        __ip_vs_conn_put(cp);

        return verdict;
}

/*
 *        Handle ICMP messages in the inside-to-outside direction (outgoing).
 *        Find any that might be relevant, check against existing connections.
 *        Currently handles error types - unreachable, quench, ttl exceeded.
 */
static int ip_vs_out_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb,
                          int *related, unsigned int hooknum)
{
        struct iphdr *iph;
        struct icmphdr        _icmph, *ic;
        struct iphdr        _ciph, *cih;        /* The ip header contained within the ICMP */
        struct ip_vs_iphdr ciph;
        struct ip_vs_conn *cp;
        struct ip_vs_protocol *pp;
        unsigned int offset, ihl;
        union nf_inet_addr snet;

        *related = 1;

        /* reassemble IP fragments */
        if (ip_is_fragment(ip_hdr(skb))) {
                if (ip_vs_gather_frags(ipvs, skb, ip_vs_defrag_user(hooknum)))
                        return NF_STOLEN;
        }

        iph = ip_hdr(skb);
        offset = ihl = iph->ihl * 4;
        ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph);
        if (ic == NULL)
                return NF_DROP;

        IP_VS_DBG(12, "Outgoing ICMP (%d,%d) %pI4->%pI4\n",
                  ic->type, ntohs(icmp_id(ic)),
                  &iph->saddr, &iph->daddr);

        /*
         * Work through seeing if this is for us.
         * These checks are supposed to be in an order that means easy
         * things are checked first to speed up processing.... however
         * this means that some packets will manage to get a long way
         * down this stack and then be rejected, but that's life.
         */
        if ((ic->type != ICMP_DEST_UNREACH) &&
            (ic->type != ICMP_SOURCE_QUENCH) &&
            (ic->type != ICMP_TIME_EXCEEDED)) {
                *related = 0;
                return NF_ACCEPT;
        }

        /* Now find the contained IP header */
        offset += sizeof(_icmph);
        cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph);
        if (cih == NULL)
                return NF_ACCEPT; /* The packet looks wrong, ignore */

        pp = ip_vs_proto_get(cih->protocol);
        if (!pp)
                return NF_ACCEPT;

        /* Is the embedded protocol header present? */
        if (unlikely(cih->frag_off & htons(IP_OFFSET) &&
                     pp->dont_defrag))
                return NF_ACCEPT;

        IP_VS_DBG_PKT(11, AF_INET, pp, skb, offset,
                      "Checking outgoing ICMP for");

        ip_vs_fill_iph_skb_icmp(AF_INET, skb, offset, true, &ciph);

        /* The embedded headers contain source and dest in reverse order */
        cp = INDIRECT_CALL_1(pp->conn_out_get, ip_vs_conn_out_get_proto,
                             ipvs, AF_INET, skb, &ciph);
        if (!cp)
                return NF_ACCEPT;

        snet.ip = iph->saddr;
        return handle_response_icmp(AF_INET, skb, &snet, cih->protocol, cp,
                                    pp, ciph.len, ihl, hooknum);
}

#ifdef CONFIG_IP_VS_IPV6
static int ip_vs_out_icmp_v6(struct netns_ipvs *ipvs, struct sk_buff *skb,
                             int *related,  unsigned int hooknum,
                             struct ip_vs_iphdr *ipvsh)
{
        struct icmp6hdr        _icmph, *ic;
        struct ip_vs_iphdr ciph = {.flags = 0, .fragoffs = 0};/*Contained IP */
        struct ip_vs_conn *cp;
        struct ip_vs_protocol *pp;
        union nf_inet_addr snet;
        unsigned int offset;

        *related = 1;
        ic = frag_safe_skb_hp(skb, ipvsh->len, sizeof(_icmph), &_icmph);
        if (ic == NULL)
                return NF_DROP;

        /*
         * Work through seeing if this is for us.
         * These checks are supposed to be in an order that means easy
         * things are checked first to speed up processing.... however
         * this means that some packets will manage to get a long way
         * down this stack and then be rejected, but that's life.
         */
        if (ic->icmp6_type & ICMPV6_INFOMSG_MASK) {
                *related = 0;
                return NF_ACCEPT;
        }
        /* Fragment header that is before ICMP header tells us that:
         * it's not an error message since they can't be fragmented.
         */
        if (ipvsh->flags & IP6_FH_F_FRAG)
                return NF_DROP;

        IP_VS_DBG(8, "Outgoing ICMPv6 (%d,%d) %pI6c->%pI6c\n",
                  ic->icmp6_type, ntohs(icmpv6_id(ic)),
                  &ipvsh->saddr, &ipvsh->daddr);

        if (!ip_vs_fill_iph_skb_icmp(AF_INET6, skb, ipvsh->len + sizeof(_icmph),
                                     true, &ciph))
                return NF_ACCEPT; /* The packet looks wrong, ignore */

        pp = ip_vs_proto_get(ciph.protocol);
        if (!pp)
                return NF_ACCEPT;

        /* The embedded headers contain source and dest in reverse order */
        cp = INDIRECT_CALL_1(pp->conn_out_get, ip_vs_conn_out_get_proto,
                             ipvs, AF_INET6, skb, &ciph);
        if (!cp)
                return NF_ACCEPT;

        snet.in6 = ciph.saddr.in6;
        offset = ciph.len;
        return handle_response_icmp(AF_INET6, skb, &snet, ciph.protocol, cp,
                                    pp, offset, sizeof(struct ipv6hdr),
                                    hooknum);
}
#endif

/*
 * Check if sctp chunc is ABORT chunk
 */
static inline int is_sctp_abort(const struct sk_buff *skb, int nh_len)
{
        struct sctp_chunkhdr *sch, schunk;
        sch = skb_header_pointer(skb, nh_len + sizeof(struct sctphdr),
                                 sizeof(schunk), &schunk);
        if (sch == NULL)
                return 0;
        if (sch->type == SCTP_CID_ABORT)
                return 1;
        return 0;
}

static inline int is_tcp_reset(const struct sk_buff *skb, int nh_len)
{
        struct tcphdr _tcph, *th;

        th = skb_header_pointer(skb, nh_len, sizeof(_tcph), &_tcph);
        if (th == NULL)
                return 0;
        return th->rst;
}

static inline bool is_new_conn(const struct sk_buff *skb,
                               struct ip_vs_iphdr *iph)
{
        switch (iph->protocol) {
        case IPPROTO_TCP: {
                struct tcphdr _tcph, *th;

                th = skb_header_pointer(skb, iph->len, sizeof(_tcph), &_tcph);
                if (th == NULL)
                        return false;
                return th->syn;
        }
        case IPPROTO_SCTP: {
                struct sctp_chunkhdr *sch, schunk;

                sch = skb_header_pointer(skb, iph->len + sizeof(struct sctphdr),
                                         sizeof(schunk), &schunk);
                if (sch == NULL)
                        return false;
                return sch->type == SCTP_CID_INIT;
        }
        default:
                return false;
        }
}

static inline bool is_new_conn_expected(const struct ip_vs_conn *cp,
                                        int conn_reuse_mode)
{
        /* Controlled (FTP DATA or persistence)? */
        if (cp->control)
                return false;

        switch (cp->protocol) {
        case IPPROTO_TCP:
                return (cp->state == IP_VS_TCP_S_TIME_WAIT) ||
                       (cp->state == IP_VS_TCP_S_CLOSE) ||
                        ((conn_reuse_mode & 2) &&
                         (cp->state == IP_VS_TCP_S_FIN_WAIT) &&
                         (cp->flags & IP_VS_CONN_F_NOOUTPUT));
        case IPPROTO_SCTP:
                return cp->state == IP_VS_SCTP_S_CLOSED;
        default:
                return false;
        }
}

/* Generic function to create new connections for outgoing RS packets
 *
 * Pre-requisites for successful connection creation:
 * 1) Virtual Service is NOT fwmark based:
 *    In fwmark-VS actual vaddr and vport are unknown to IPVS
 * 2) Real Server and Virtual Service were NOT configured without port:
 *    This is to allow match of different VS to the same RS ip-addr
 */
struct ip_vs_conn *ip_vs_new_conn_out(struct ip_vs_service *svc,
                                      struct ip_vs_dest *dest,
                                      struct sk_buff *skb,
                                      const struct ip_vs_iphdr *iph,
                                      __be16 dport,
                                      __be16 cport)
{
        struct ip_vs_conn_param param;
        struct ip_vs_conn *ct = NULL, *cp = NULL;
        const union nf_inet_addr *vaddr, *daddr, *caddr;
        union nf_inet_addr snet;
        __be16 vport;
        unsigned int flags;

        vaddr = &svc->addr;
        vport = svc->port;
        daddr = &iph->saddr;
        caddr = &iph->daddr;

        /* check pre-requisites are satisfied */
        if (svc->fwmark)
                return NULL;
        if (!vport || !dport)
                return NULL;

        /* for persistent service first create connection template */
        if (svc->flags & IP_VS_SVC_F_PERSISTENT) {
                /* apply netmask the same way ingress-side does */
#ifdef CONFIG_IP_VS_IPV6
                if (svc->af == AF_INET6)
                        ipv6_addr_prefix(&snet.in6, &caddr->in6,
                                         (__force __u32)svc->netmask);
                else
#endif
                        snet.ip = caddr->ip & svc->netmask;
                /* fill params and create template if not existent */
                if (ip_vs_conn_fill_param_persist(svc, skb, iph->protocol,
                                                  &snet, 0, vaddr,
                                                  vport, &param) < 0)
                        return NULL;
                ct = ip_vs_ct_in_get(&param);
                /* check if template exists and points to the same dest */
                if (!ct || !ip_vs_check_template(ct, dest)) {
                        ct = ip_vs_conn_new(&param, dest->af, daddr, dport,
                                            IP_VS_CONN_F_TEMPLATE, dest, 0);
                        if (!ct) {
                                kfree(param.pe_data);
                                return NULL;
                        }
                        ct->timeout = svc->timeout;
                } else {
                        kfree(param.pe_data);
                }
        }

        /* connection flags */
        flags = ((svc->flags & IP_VS_SVC_F_ONEPACKET) &&
                 iph->protocol == IPPROTO_UDP) ? IP_VS_CONN_F_ONE_PACKET : 0;
        /* create connection */
        ip_vs_conn_fill_param(svc->ipvs, svc->af, iph->protocol,
                              caddr, cport, vaddr, vport, &param);
        cp = ip_vs_conn_new(&param, dest->af, daddr, dport, flags, dest, 0);
        if (!cp) {
                if (ct)
                        ip_vs_conn_put(ct);
                return NULL;
        }
        if (ct) {
                ip_vs_control_add(cp, ct);
                ip_vs_conn_put(ct);
        }
        ip_vs_conn_stats(cp, svc);

        /* return connection (will be used to handle outgoing packet) */
        IP_VS_DBG_BUF(6, "New connection RS-initiated:%c c:%s:%u v:%s:%u "
                      "d:%s:%u conn->flags:%X conn->refcnt:%d\n",
                      ip_vs_fwd_tag(cp),
                      IP_VS_DBG_ADDR(cp->af, &cp->caddr), ntohs(cp->cport),
                      IP_VS_DBG_ADDR(cp->af, &cp->vaddr), ntohs(cp->vport),
                      IP_VS_DBG_ADDR(cp->af, &cp->daddr), ntohs(cp->dport),
                      cp->flags, refcount_read(&cp->refcnt));
        return cp;
}

/* Handle outgoing packets which are considered requests initiated by
 * real servers, so that subsequent responses from external client can be
 * routed to the right real server.
 * Used also for outgoing responses in OPS mode.
 *
 * Connection management is handled by persistent-engine specific callback.
 */
static struct ip_vs_conn *__ip_vs_rs_conn_out(unsigned int hooknum,
                                              struct netns_ipvs *ipvs,
                                              int af, struct sk_buff *skb,
                                              const struct ip_vs_iphdr *iph)
{
        struct ip_vs_dest *dest;
        struct ip_vs_conn *cp = NULL;
        __be16 _ports[2], *pptr;

        if (hooknum == NF_INET_LOCAL_IN)
                return NULL;

        pptr = frag_safe_skb_hp(skb, iph->len,
                                sizeof(_ports), _ports);
        if (!pptr)
                return NULL;

        dest = ip_vs_find_real_service(ipvs, af, iph->protocol,
                                       &iph->saddr, pptr[0]);
        if (dest) {
                struct ip_vs_service *svc;
                struct ip_vs_pe *pe;

                svc = rcu_dereference(dest->svc);
                if (svc) {
                        pe = rcu_dereference(svc->pe);
                        if (pe && pe->conn_out)
                                cp = pe->conn_out(svc, dest, skb, iph,
                                                  pptr[0], pptr[1]);
                }
        }

        return cp;
}

/* Handle response packets: rewrite addresses and send away...
 */
static unsigned int
handle_response(int af, struct sk_buff *skb, struct ip_vs_proto_data *pd,
                struct ip_vs_conn *cp, struct ip_vs_iphdr *iph,
                unsigned int hooknum)
{
        struct ip_vs_protocol *pp = pd->pp;

        if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ)
                goto after_nat;

        IP_VS_DBG_PKT(11, af, pp, skb, iph->off, "Outgoing packet");

        if (skb_ensure_writable(skb, iph->len))
                goto drop;

        /* mangle the packet */
        if (pp->snat_handler &&
            !SNAT_CALL(pp->snat_handler, skb, pp, cp, iph))
                goto drop;

#ifdef CONFIG_IP_VS_IPV6
        if (af == AF_INET6)
                ipv6_hdr(skb)->saddr = cp->vaddr.in6;
        else
#endif
        {
                ip_hdr(skb)->saddr = cp->vaddr.ip;
                ip_send_check(ip_hdr(skb));
        }

        /*
         * nf_iterate does not expect change in the skb->dst->dev.
         * It looks like it is not fatal to enable this code for hooks
         * where our handlers are at the end of the chain list and
         * when all next handlers use skb->dst->dev and not outdev.
         * It will definitely route properly the inout NAT traffic
         * when multiple paths are used.
         */

        /* For policy routing, packets originating from this
         * machine itself may be routed differently to packets
         * passing through.  We want this packet to be routed as
         * if it came from this machine itself.  So re-compute
         * the routing information.
         */
        if (ip_vs_route_me_harder(cp->ipvs, af, skb, hooknum))
                goto drop;

        IP_VS_DBG_PKT(10, af, pp, skb, iph->off, "After SNAT");

after_nat:
        ip_vs_out_stats(cp, skb);
        ip_vs_set_state(cp, IP_VS_DIR_OUTPUT, skb, pd);
        skb->ipvs_property = 1;
        if (!(cp->flags & IP_VS_CONN_F_NFCT))
                ip_vs_notrack(skb);
        else
                ip_vs_update_conntrack(skb, cp, 0);
        ip_vs_conn_put(cp);

        return NF_ACCEPT;

drop:
        ip_vs_conn_put(cp);
        kfree_skb(skb);
        return NF_STOLEN;
}

/*
 *        Check if outgoing packet belongs to the established ip_vs_conn.
 */
static unsigned int
ip_vs_out_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *state)
{
        struct netns_ipvs *ipvs = net_ipvs(state->net);
        unsigned int hooknum = state->hook;
        struct ip_vs_iphdr iph;
        struct ip_vs_protocol *pp;
        struct ip_vs_proto_data *pd;
        struct ip_vs_conn *cp;
        int af = state->pf;
        struct sock *sk;

        /* Already marked as IPVS request or reply? */
        if (skb->ipvs_property)
                return NF_ACCEPT;

        sk = skb_to_full_sk(skb);
        /* Bad... Do not break raw sockets */
        if (unlikely(sk && hooknum == NF_INET_LOCAL_OUT &&
                     af == AF_INET)) {

                if (sk->sk_family == PF_INET && inet_test_bit(NODEFRAG, sk))
                        return NF_ACCEPT;
        }

        if (unlikely(!skb_dst(skb)))
                return NF_ACCEPT;

        if (!ipvs->enable)
                return NF_ACCEPT;

        ip_vs_fill_iph_skb(af, skb, false, &iph);
#ifdef CONFIG_IP_VS_IPV6
        if (af == AF_INET6) {
                if (unlikely(iph.protocol == IPPROTO_ICMPV6)) {
                        int related;
                        int verdict = ip_vs_out_icmp_v6(ipvs, skb, &related,
                                                        hooknum, &iph);

                        if (related)
                                return verdict;
                }
        } else
#endif
                if (unlikely(iph.protocol == IPPROTO_ICMP)) {
                        int related;
                        int verdict = ip_vs_out_icmp(ipvs, skb, &related, hooknum);

                        if (related)
                                return verdict;
                }

        pd = ip_vs_proto_data_get(ipvs, iph.protocol);
        if (unlikely(!pd))
                return NF_ACCEPT;
        pp = pd->pp;

        /* reassemble IP fragments */
#ifdef CONFIG_IP_VS_IPV6
        if (af == AF_INET)
#endif
                if (unlikely(ip_is_fragment(ip_hdr(skb)) && !pp->dont_defrag)) {
                        if (ip_vs_gather_frags(ipvs, skb,
                                               ip_vs_defrag_user(hooknum)))
                                return NF_STOLEN;

                        ip_vs_fill_iph_skb(AF_INET, skb, false, &iph);
                }

        /*
         * Check if the packet belongs to an existing entry
         */
        cp = INDIRECT_CALL_1(pp->conn_out_get, ip_vs_conn_out_get_proto,
                             ipvs, af, skb, &iph);

        if (likely(cp))
                return handle_response(af, skb, pd, cp, &iph, hooknum);

        /* Check for real-server-started requests */
        if (atomic_read(&ipvs->conn_out_counter)) {
                /* Currently only for UDP:
                 * connection oriented protocols typically use
                 * ephemeral ports for outgoing connections, so
                 * related incoming responses would not match any VS
                 */
                if (pp->protocol == IPPROTO_UDP) {
                        cp = __ip_vs_rs_conn_out(hooknum, ipvs, af, skb, &iph);
                        if (likely(cp))
                                return handle_response(af, skb, pd, cp, &iph,
                                                       hooknum);
                }
        }

        if (sysctl_nat_icmp_send(ipvs) &&
            (pp->protocol == IPPROTO_TCP ||
             pp->protocol == IPPROTO_UDP ||
             pp->protocol == IPPROTO_SCTP)) {
                __be16 _ports[2], *pptr;

                pptr = frag_safe_skb_hp(skb, iph.len,
                                         sizeof(_ports), _ports);
                if (pptr == NULL)
                        return NF_ACCEPT;        /* Not for me */
                if (ip_vs_has_real_service(ipvs, af, iph.protocol, &iph.saddr,
                                           pptr[0])) {
                        /*
                         * Notify the real server: there is no
                         * existing entry if it is not RST
                         * packet or not TCP packet.
                         */
                        if ((iph.protocol != IPPROTO_TCP &&
                             iph.protocol != IPPROTO_SCTP)
                             || ((iph.protocol == IPPROTO_TCP
                                  && !is_tcp_reset(skb, iph.len))
                                 || (iph.protocol == IPPROTO_SCTP
                                        && !is_sctp_abort(skb,
                                                iph.len)))) {
#ifdef CONFIG_IP_VS_IPV6
                                if (af == AF_INET6) {
                                        if (!skb->dev)
                                                skb->dev = ipvs->net->loopback_dev;
                                        icmpv6_send(skb,
                                                    ICMPV6_DEST_UNREACH,
                                                    ICMPV6_PORT_UNREACH,
                                                    0);
                                } else
#endif
                                        icmp_send(skb,
                                                  ICMP_DEST_UNREACH,
                                                  ICMP_PORT_UNREACH, 0);
                                return NF_DROP;
                        }
                }
        }

        IP_VS_DBG_PKT(12, af, pp, skb, iph.off,
                      "ip_vs_out: packet continues traversal as normal");
        return NF_ACCEPT;
}

static unsigned int
ip_vs_try_to_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb,
                      struct ip_vs_proto_data *pd,
                      int *verdict, struct ip_vs_conn **cpp,
                      struct ip_vs_iphdr *iph)
{
        struct ip_vs_protocol *pp = pd->pp;

        if (!iph->fragoffs) {
                /* No (second) fragments need to enter here, as nf_defrag_ipv6
                 * replayed fragment zero will already have created the cp
                 */

                /* Schedule and create new connection entry into cpp */
                if (!pp->conn_schedule(ipvs, af, skb, pd, verdict, cpp, iph))
                        return 0;
        }

        if (unlikely(!*cpp)) {
                /* sorry, all this trouble for a no-hit :) */
                IP_VS_DBG_PKT(12, af, pp, skb, iph->off,
                              "ip_vs_in: packet continues traversal as normal");

                /* Fragment couldn't be mapped to a conn entry */
                if (iph->fragoffs)
                        IP_VS_DBG_PKT(7, af, pp, skb, iph->off,
                                      "unhandled fragment");

                *verdict = NF_ACCEPT;
                return 0;
        }

        return 1;
}

/* Check the UDP tunnel and return its header length */
static int ipvs_udp_decap(struct netns_ipvs *ipvs, struct sk_buff *skb,
                          unsigned int offset, __u16 af,
                          const union nf_inet_addr *daddr, __u8 *proto)
{
        struct udphdr _udph, *udph;
        struct ip_vs_dest *dest;

        udph = skb_header_pointer(skb, offset, sizeof(_udph), &_udph);
        if (!udph)
                goto unk;
        offset += sizeof(struct udphdr);
        dest = ip_vs_find_tunnel(ipvs, af, daddr, udph->dest);
        if (!dest)
                goto unk;
        if (dest->tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) {
                struct guehdr _gueh, *gueh;

                gueh = skb_header_pointer(skb, offset, sizeof(_gueh), &_gueh);
                if (!gueh)
                        goto unk;
                if (gueh->control != 0 || gueh->version != 0)
                        goto unk;
                /* Later we can support also IPPROTO_IPV6 */
                if (gueh->proto_ctype != IPPROTO_IPIP)
                        goto unk;
                *proto = gueh->proto_ctype;
                return sizeof(struct udphdr) + sizeof(struct guehdr) +
                       (gueh->hlen << 2);
        }

unk:
        return 0;
}

/* Check the GRE tunnel and return its header length */
static int ipvs_gre_decap(struct netns_ipvs *ipvs, struct sk_buff *skb,
                          unsigned int offset, __u16 af,
                          const union nf_inet_addr *daddr, __u8 *proto)
{
        struct gre_base_hdr _greh, *greh;
        struct ip_vs_dest *dest;

        greh = skb_header_pointer(skb, offset, sizeof(_greh), &_greh);
        if (!greh)
                goto unk;
        dest = ip_vs_find_tunnel(ipvs, af, daddr, 0);
        if (!dest)
                goto unk;
        if (dest->tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GRE) {
                IP_TUNNEL_DECLARE_FLAGS(flags);
                __be16 type;

                /* Only support version 0 and C (csum) */
                if ((greh->flags & ~GRE_CSUM) != 0)
                        goto unk;
                type = greh->protocol;
                /* Later we can support also IPPROTO_IPV6 */
                if (type != htons(ETH_P_IP))
                        goto unk;
                *proto = IPPROTO_IPIP;

                gre_flags_to_tnl_flags(flags, greh->flags);

                return gre_calc_hlen(flags);
        }

unk:
        return 0;
}

/*
 *        Handle ICMP messages in the outside-to-inside direction (incoming).
 *        Find any that might be relevant, check against existing connections,
 *        forward to the right destination host if relevant.
 *        Currently handles error types - unreachable, quench, ttl exceeded.
 */
static int
ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related,
              unsigned int hooknum)
{
        struct iphdr *iph;
        struct icmphdr        _icmph, *ic;
        struct iphdr        _ciph, *cih;        /* The ip header contained within the ICMP */
        struct ip_vs_iphdr ciph;
        struct ip_vs_conn *cp;
        struct ip_vs_protocol *pp;
        struct ip_vs_proto_data *pd;
        unsigned int offset, offset2, ihl, verdict;
        bool tunnel, new_cp = false;
        union nf_inet_addr *raddr;
        char *outer_proto = "IPIP";

        *related = 1;

        /* reassemble IP fragments */
        if (ip_is_fragment(ip_hdr(skb))) {
                if (ip_vs_gather_frags(ipvs, skb, ip_vs_defrag_user(hooknum)))
                        return NF_STOLEN;
        }

        iph = ip_hdr(skb);
        offset = ihl = iph->ihl * 4;
        ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph);
        if (ic == NULL)
                return NF_DROP;

        IP_VS_DBG(12, "Incoming ICMP (%d,%d) %pI4->%pI4\n",
                  ic->type, ntohs(icmp_id(ic)),
                  &iph->saddr, &iph->daddr);

        /*
         * Work through seeing if this is for us.
         * These checks are supposed to be in an order that means easy
         * things are checked first to speed up processing.... however
         * this means that some packets will manage to get a long way
         * down this stack and then be rejected, but that's life.
         */
        if ((ic->type != ICMP_DEST_UNREACH) &&
            (ic->type != ICMP_SOURCE_QUENCH) &&
            (ic->type != ICMP_TIME_EXCEEDED)) {
                *related = 0;
                return NF_ACCEPT;
        }

        /* Now find the contained IP header */
        offset += sizeof(_icmph);
        cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph);
        if (cih == NULL)
                return NF_ACCEPT; /* The packet looks wrong, ignore */
        raddr = (union nf_inet_addr *)&cih->daddr;

        /* Special case for errors for IPIP/UDP/GRE tunnel packets */
        tunnel = false;
        if (cih->protocol == IPPROTO_IPIP) {
                struct ip_vs_dest *dest;

                if (unlikely(cih->frag_off & htons(IP_OFFSET)))
                        return NF_ACCEPT;
                /* Error for our IPIP must arrive at LOCAL_IN */
                if (!(skb_rtable(skb)->rt_flags & RTCF_LOCAL))
                        return NF_ACCEPT;
                dest = ip_vs_find_tunnel(ipvs, AF_INET, raddr, 0);
                /* Only for known tunnel */
                if (!dest || dest->tun_type != IP_VS_CONN_F_TUNNEL_TYPE_IPIP)
                        return NF_ACCEPT;
                offset += cih->ihl * 4;
                cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph);
                if (cih == NULL)
                        return NF_ACCEPT; /* The packet looks wrong, ignore */
                tunnel = true;
        } else if ((cih->protocol == IPPROTO_UDP ||        /* Can be UDP encap */
                    cih->protocol == IPPROTO_GRE) &&        /* Can be GRE encap */
                   /* Error for our tunnel must arrive at LOCAL_IN */
                   (skb_rtable(skb)->rt_flags & RTCF_LOCAL)) {
                __u8 iproto;
                int ulen;

                /* Non-first fragment has no UDP/GRE header */
                if (unlikely(cih->frag_off & htons(IP_OFFSET)))
                        return NF_ACCEPT;
                offset2 = offset + cih->ihl * 4;
                if (cih->protocol == IPPROTO_UDP) {
                        ulen = ipvs_udp_decap(ipvs, skb, offset2, AF_INET,
                                              raddr, &iproto);
                        outer_proto = "UDP";
                } else {
                        ulen = ipvs_gre_decap(ipvs, skb, offset2, AF_INET,
                                              raddr, &iproto);
                        outer_proto = "GRE";
                }
                if (ulen > 0) {
                        /* Skip IP and UDP/GRE tunnel headers */
                        offset = offset2 + ulen;
                        /* Now we should be at the original IP header */
                        cih = skb_header_pointer(skb, offset, sizeof(_ciph),
                                                 &_ciph);
                        if (cih && cih->version == 4 && cih->ihl >= 5 &&
                            iproto == IPPROTO_IPIP)
                                tunnel = true;
                        else
                                return NF_ACCEPT;
                }
        }

        pd = ip_vs_proto_data_get(ipvs, cih->protocol);
        if (!pd)
                return NF_ACCEPT;
        pp = pd->pp;

        /* Is the embedded protocol header present? */
        if (unlikely(cih->frag_off & htons(IP_OFFSET) &&
                     pp->dont_defrag))
                return NF_ACCEPT;

        IP_VS_DBG_PKT(11, AF_INET, pp, skb, offset,
                      "Checking incoming ICMP for");

        offset2 = offset;
        ip_vs_fill_iph_skb_icmp(AF_INET, skb, offset, !tunnel, &ciph);
        offset = ciph.len;

        /* The embedded headers contain source and dest in reverse order.
         * For IPIP/UDP/GRE tunnel this is error for request, not for reply.
         */
        cp = INDIRECT_CALL_1(pp->conn_in_get, ip_vs_conn_in_get_proto,
                             ipvs, AF_INET, skb, &ciph);

        if (!cp) {
                int v;

                if (tunnel || !sysctl_schedule_icmp(ipvs))
                        return NF_ACCEPT;

                if (!ip_vs_try_to_schedule(ipvs, AF_INET, skb, pd, &v, &cp, &ciph))
                        return v;
                new_cp = true;
        }

        verdict = NF_DROP;

        /* Ensure the checksum is correct */
        if (!skb_csum_unnecessary(skb) && ip_vs_checksum_complete(skb, ihl)) {
                /* Failed checksum! */
                IP_VS_DBG(1, "Incoming ICMP: failed checksum from %pI4!\n",
                          &iph->saddr);
                goto out;
        }

        if (tunnel) {
                __be32 info = ic->un.gateway;
                __u8 type = ic->type;
                __u8 code = ic->code;

                /* Update the MTU */
                if (ic->type == ICMP_DEST_UNREACH &&
                    ic->code == ICMP_FRAG_NEEDED) {
                        struct ip_vs_dest *dest = cp->dest;
                        u32 mtu = ntohs(ic->un.frag.mtu);
                        __be16 frag_off = cih->frag_off;

                        /* Strip outer IP and ICMP, go to IPIP/UDP/GRE header */
                        if (pskb_pull(skb, ihl + sizeof(_icmph)) == NULL)
                                goto ignore_tunnel;
                        offset2 -= ihl + sizeof(_icmph);
                        skb_reset_network_header(skb);
                        IP_VS_DBG(12, "ICMP for %s %pI4->%pI4: mtu=%u\n",
                                  outer_proto, &ip_hdr(skb)->saddr,
                                  &ip_hdr(skb)->daddr, mtu);
                        ipv4_update_pmtu(skb, ipvs->net, mtu, 0, 0);
                        /* Client uses PMTUD? */
                        if (!(frag_off & htons(IP_DF)))
                                goto ignore_tunnel;
                        /* Prefer the resulting PMTU */
                        if (dest) {
                                struct ip_vs_dest_dst *dest_dst;

                                dest_dst = rcu_dereference(dest->dest_dst);
                                if (dest_dst)
                                        mtu = dst_mtu(dest_dst->dst_cache);
                        }
                        if (mtu > 68 + sizeof(struct iphdr))
                                mtu -= sizeof(struct iphdr);
                        info = htonl(mtu);
                }
                /* Strip outer IP, ICMP and IPIP/UDP/GRE, go to IP header of
                 * original request.
                 */
                if (pskb_pull(skb, offset2) == NULL)
                        goto ignore_tunnel;
                skb_reset_network_header(skb);
                IP_VS_DBG(12, "Sending ICMP for %pI4->%pI4: t=%u, c=%u, i=%u\n",
                        &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
                        type, code, ntohl(info));
                icmp_send(skb, type, code, info);
                /* ICMP can be shorter but anyways, account it */
                ip_vs_out_stats(cp, skb);

ignore_tunnel:
                consume_skb(skb);
                verdict = NF_STOLEN;
                goto out;
        }

        /* do the statistics and put it back */
        ip_vs_in_stats(cp, skb);
        if (IPPROTO_TCP == cih->protocol || IPPROTO_UDP == cih->protocol ||
            IPPROTO_SCTP == cih->protocol)
                offset += 2 * sizeof(__u16);
        verdict = ip_vs_icmp_xmit(skb, cp, pp, offset, hooknum, &ciph);

out:
        if (likely(!new_cp))
                __ip_vs_conn_put(cp);
        else
                ip_vs_conn_put(cp);

        return verdict;
}

#ifdef CONFIG_IP_VS_IPV6
static int ip_vs_in_icmp_v6(struct netns_ipvs *ipvs, struct sk_buff *skb,
                            int *related, unsigned int hooknum,
                            struct ip_vs_iphdr *iph)
{
        struct icmp6hdr        _icmph, *ic;
        struct ip_vs_iphdr ciph = {.flags = 0, .fragoffs = 0};/*Contained IP */
        struct ip_vs_conn *cp;
        struct ip_vs_protocol *pp;
        struct ip_vs_proto_data *pd;
        unsigned int offset, verdict;
        bool new_cp = false;

        *related = 1;

        ic = frag_safe_skb_hp(skb, iph->len, sizeof(_icmph), &_icmph);
        if (ic == NULL)
                return NF_DROP;

        /*
         * Work through seeing if this is for us.
         * These checks are supposed to be in an order that means easy
         * things are checked first to speed up processing.... however
         * this means that some packets will manage to get a long way
         * down this stack and then be rejected, but that's life.
         */
        if (ic->icmp6_type & ICMPV6_INFOMSG_MASK) {
                *related = 0;
                return NF_ACCEPT;
        }
        /* Fragment header that is before ICMP header tells us that:
         * it's not an error message since they can't be fragmented.
         */
        if (iph->flags & IP6_FH_F_FRAG)
                return NF_DROP;

        IP_VS_DBG(8, "Incoming ICMPv6 (%d,%d) %pI6c->%pI6c\n",
                  ic->icmp6_type, ntohs(icmpv6_id(ic)),
                  &iph->saddr, &iph->daddr);

        offset = iph->len + sizeof(_icmph);
        if (!ip_vs_fill_iph_skb_icmp(AF_INET6, skb, offset, true, &ciph))
                return NF_ACCEPT;

        pd = ip_vs_proto_data_get(ipvs, ciph.protocol);
        if (!pd)
                return NF_ACCEPT;
        pp = pd->pp;

        /* Cannot handle fragmented embedded protocol */
        if (ciph.fragoffs)
                return NF_ACCEPT;

        IP_VS_DBG_PKT(11, AF_INET6, pp, skb, offset,
                      "Checking incoming ICMPv6 for");

        /* The embedded headers contain source and dest in reverse order
         * if not from localhost
         */
        cp = INDIRECT_CALL_1(pp->conn_in_get, ip_vs_conn_in_get_proto,
                             ipvs, AF_INET6, skb, &ciph);

        if (!cp) {
                int v;

                if (!sysctl_schedule_icmp(ipvs))
                        return NF_ACCEPT;

                if (!ip_vs_try_to_schedule(ipvs, AF_INET6, skb, pd, &v, &cp, &ciph))
                        return v;

                new_cp = true;
        }

        /* VS/TUN, VS/DR and LOCALNODE just let it go */
        if ((hooknum == NF_INET_LOCAL_OUT) &&
            (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ)) {
                verdict = NF_ACCEPT;
                goto out;
        }

        /* do the statistics and put it back */
        ip_vs_in_stats(cp, skb);

        /* Need to mangle contained IPv6 header in ICMPv6 packet */
        offset = ciph.len;
        if (IPPROTO_TCP == ciph.protocol || IPPROTO_UDP == ciph.protocol ||
            IPPROTO_SCTP == ciph.protocol)
                offset += 2 * sizeof(__u16); /* Also mangle ports */

        verdict = ip_vs_icmp_xmit_v6(skb, cp, pp, offset, hooknum, &ciph);

out:
        if (likely(!new_cp))
                __ip_vs_conn_put(cp);
        else
                ip_vs_conn_put(cp);

        return verdict;
}
#endif


/*
 *        Check if it's for virtual services, look it up,
 *        and send it on its way...
 */
static unsigned int
ip_vs_in_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *state)
{
        struct netns_ipvs *ipvs = net_ipvs(state->net);
        unsigned int hooknum = state->hook;
        struct ip_vs_iphdr iph;
        struct ip_vs_protocol *pp;
        struct ip_vs_proto_data *pd;
        struct ip_vs_conn *cp;
        int ret, pkts;
        struct sock *sk;
        int af = state->pf;

        /* Already marked as IPVS request or reply? */
        if (skb->ipvs_property)
                return NF_ACCEPT;

        /*
         *        Big tappo:
         *        - remote client: only PACKET_HOST
         *        - route: used for struct net when skb->dev is unset
         */
        if (unlikely((skb->pkt_type != PACKET_HOST &&
                      hooknum != NF_INET_LOCAL_OUT) ||
                     !skb_dst(skb))) {
                ip_vs_fill_iph_skb(af, skb, false, &iph);
                IP_VS_DBG_BUF(12, "packet type=%d proto=%d daddr=%s"
                              " ignored in hook %u\n",
                              skb->pkt_type, iph.protocol,
                              IP_VS_DBG_ADDR(af, &iph.daddr), hooknum);
                return NF_ACCEPT;
        }
        /* ipvs enabled in this netns ? */
        if (unlikely(sysctl_backup_only(ipvs) || !ipvs->enable))
                return NF_ACCEPT;

        ip_vs_fill_iph_skb(af, skb, false, &iph);

        /* Bad... Do not break raw sockets */
        sk = skb_to_full_sk(skb);
        if (unlikely(sk && hooknum == NF_INET_LOCAL_OUT &&
                     af == AF_INET)) {

                if (sk->sk_family == PF_INET && inet_test_bit(NODEFRAG, sk))
                        return NF_ACCEPT;
        }

#ifdef CONFIG_IP_VS_IPV6
        if (af == AF_INET6) {
                if (unlikely(iph.protocol == IPPROTO_ICMPV6)) {
                        int related;
                        int verdict = ip_vs_in_icmp_v6(ipvs, skb, &related,
                                                       hooknum, &iph);

                        if (related)
                                return verdict;
                }
        } else
#endif
                if (unlikely(iph.protocol == IPPROTO_ICMP)) {
                        int related;
                        int verdict = ip_vs_in_icmp(ipvs, skb, &related,
                                                    hooknum);

                        if (related)
                                return verdict;
                }

        /* Protocol supported? */
        pd = ip_vs_proto_data_get(ipvs, iph.protocol);
        if (unlikely(!pd)) {
                /* The only way we'll see this packet again is if it's
                 * encapsulated, so mark it with ipvs_property=1 so we
                 * skip it if we're ignoring tunneled packets
                 */
                if (sysctl_ignore_tunneled(ipvs))
                        skb->ipvs_property = 1;

                return NF_ACCEPT;
        }
        pp = pd->pp;
        /*
         * Check if the packet belongs to an existing connection entry
         */
        cp = INDIRECT_CALL_1(pp->conn_in_get, ip_vs_conn_in_get_proto,
                             ipvs, af, skb, &iph);

        if (!iph.fragoffs && is_new_conn(skb, &iph) && cp) {
                int conn_reuse_mode = sysctl_conn_reuse_mode(ipvs);
                bool old_ct = false, resched = false;

                if (unlikely(sysctl_expire_nodest_conn(ipvs)) && cp->dest &&
                    unlikely(!atomic_read(&cp->dest->weight))) {
                        resched = true;
                        old_ct = ip_vs_conn_uses_old_conntrack(cp, skb);
                } else if (conn_reuse_mode &&
                           is_new_conn_expected(cp, conn_reuse_mode)) {
                        old_ct = ip_vs_conn_uses_old_conntrack(cp, skb);
                        if (!atomic_read(&cp->n_control)) {
                                resched = true;
                        } else {
                                /* Do not reschedule controlling connection
                                 * that uses conntrack while it is still
                                 * referenced by controlled connection(s).
                                 */
                                resched = !old_ct;
                        }
                }

                if (resched) {
                        if (!old_ct)
                                cp->flags &= ~IP_VS_CONN_F_NFCT;
                        if (!atomic_read(&cp->n_control))
                                ip_vs_conn_expire_now(cp);
                        __ip_vs_conn_put(cp);
                        if (old_ct)
                                return NF_DROP;
                        cp = NULL;
                }
        }

        /* Check the server status */
        if (cp && cp->dest && !(cp->dest->flags & IP_VS_DEST_F_AVAILABLE)) {
                /* the destination server is not available */
                if (sysctl_expire_nodest_conn(ipvs)) {
                        bool old_ct = ip_vs_conn_uses_old_conntrack(cp, skb);

                        if (!old_ct)
                                cp->flags &= ~IP_VS_CONN_F_NFCT;

                        ip_vs_conn_expire_now(cp);
                        __ip_vs_conn_put(cp);
                        if (old_ct)
                                return NF_DROP;
                        cp = NULL;
                } else {
                        __ip_vs_conn_put(cp);
                        return NF_DROP;
                }
        }

        if (unlikely(!cp)) {
                int v;

                if (!ip_vs_try_to_schedule(ipvs, af, skb, pd, &v, &cp, &iph))
                        return v;
        }

        IP_VS_DBG_PKT(11, af, pp, skb, iph.off, "Incoming packet");

        ip_vs_in_stats(cp, skb);
        ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pd);
        if (cp->packet_xmit)
                ret = cp->packet_xmit(skb, cp, pp, &iph);
                /* do not touch skb anymore */
        else {
                IP_VS_DBG_RL("warning: packet_xmit is null");
                ret = NF_ACCEPT;
        }

        /* Increase its packet counter and check if it is needed
         * to be synchronized
         *
         * Sync connection if it is about to close to
         * encorage the standby servers to update the connections timeout
         *
         * For ONE_PKT let ip_vs_sync_conn() do the filter work.
         */

        if (cp->flags & IP_VS_CONN_F_ONE_PACKET)
                pkts = sysctl_sync_threshold(ipvs);
        else
                pkts = atomic_inc_return(&cp->in_pkts);

        if (ipvs->sync_state & IP_VS_STATE_MASTER)
                ip_vs_sync_conn(ipvs, cp, pkts);
        else if ((cp->flags & IP_VS_CONN_F_ONE_PACKET) && cp->control)
                /* increment is done inside ip_vs_sync_conn too */
                atomic_inc(&cp->control->in_pkts);

        ip_vs_conn_put(cp);
        return ret;
}

/*
 *        It is hooked at the NF_INET_FORWARD chain, in order to catch ICMP
 *      related packets destined for 0.0.0.0/0.
 *      When fwmark-based virtual service is used, such as transparent
 *      cache cluster, TCP packets can be marked and routed to ip_vs_in,
 *      but ICMP destined for 0.0.0.0/0 cannot not be easily marked and
 *      sent to ip_vs_in_icmp. So, catch them at the NF_INET_FORWARD chain
 *      and send them to ip_vs_in_icmp.
 */
static unsigned int
ip_vs_forward_icmp(void *priv, struct sk_buff *skb,
                   const struct nf_hook_state *state)
{
        struct netns_ipvs *ipvs = net_ipvs(state->net);
        int r;

        /* ipvs enabled in this netns ? */
        if (unlikely(sysctl_backup_only(ipvs) || !ipvs->enable))
                return NF_ACCEPT;

        if (state->pf == NFPROTO_IPV4) {
                if (ip_hdr(skb)->protocol != IPPROTO_ICMP)
                        return NF_ACCEPT;
#ifdef CONFIG_IP_VS_IPV6
        } else {
                struct ip_vs_iphdr iphdr;

                ip_vs_fill_iph_skb(AF_INET6, skb, false, &iphdr);

                if (iphdr.protocol != IPPROTO_ICMPV6)
                        return NF_ACCEPT;

                return ip_vs_in_icmp_v6(ipvs, skb, &r, state->hook, &iphdr);
#endif
        }

        return ip_vs_in_icmp(ipvs, skb, &r, state->hook);
}

static const struct nf_hook_ops ip_vs_ops4[] = {
        /* After packet filtering, change source only for VS/NAT */
        {
                .hook                = ip_vs_out_hook,
                .pf                = NFPROTO_IPV4,
                .hooknum        = NF_INET_LOCAL_IN,
                .priority        = NF_IP_PRI_NAT_SRC - 2,
        },
        /* After packet filtering, forward packet through VS/DR, VS/TUN,
         * or VS/NAT(change destination), so that filtering rules can be
         * applied to IPVS. */
        {
                .hook                = ip_vs_in_hook,
                .pf                = NFPROTO_IPV4,
                .hooknum        = NF_INET_LOCAL_IN,
                .priority        = NF_IP_PRI_NAT_SRC - 1,
        },
        /* Before ip_vs_in, change source only for VS/NAT */
        {
                .hook                = ip_vs_out_hook,
                .pf                = NFPROTO_IPV4,
                .hooknum        = NF_INET_LOCAL_OUT,
                .priority        = NF_IP_PRI_NAT_DST + 1,
        },
        /* After mangle, schedule and forward local requests */
        {
                .hook                = ip_vs_in_hook,
                .pf                = NFPROTO_IPV4,
                .hooknum        = NF_INET_LOCAL_OUT,
                .priority        = NF_IP_PRI_NAT_DST + 2,
        },
        /* After packet filtering (but before ip_vs_out_icmp), catch icmp
         * destined for 0.0.0.0/0, which is for incoming IPVS connections */
        {
                .hook                = ip_vs_forward_icmp,
                .pf                = NFPROTO_IPV4,
                .hooknum        = NF_INET_FORWARD,
                .priority        = 99,
        },
        /* After packet filtering, change source only for VS/NAT */
        {
                .hook                = ip_vs_out_hook,
                .pf                = NFPROTO_IPV4,
                .hooknum        = NF_INET_FORWARD,
                .priority        = 100,
        },
};

#ifdef CONFIG_IP_VS_IPV6
static const struct nf_hook_ops ip_vs_ops6[] = {
        /* After packet filtering, change source only for VS/NAT */
        {
                .hook                = ip_vs_out_hook,
                .pf                = NFPROTO_IPV6,
                .hooknum        = NF_INET_LOCAL_IN,
                .priority        = NF_IP6_PRI_NAT_SRC - 2,
        },
        /* After packet filtering, forward packet through VS/DR, VS/TUN,
         * or VS/NAT(change destination), so that filtering rules can be
         * applied to IPVS. */
        {
                .hook                = ip_vs_in_hook,
                .pf                = NFPROTO_IPV6,
                .hooknum        = NF_INET_LOCAL_IN,
                .priority        = NF_IP6_PRI_NAT_SRC - 1,
        },
        /* Before ip_vs_in, change source only for VS/NAT */
        {
                .hook                = ip_vs_out_hook,
                .pf                = NFPROTO_IPV6,
                .hooknum        = NF_INET_LOCAL_OUT,
                .priority        = NF_IP6_PRI_NAT_DST + 1,
        },
        /* After mangle, schedule and forward local requests */
        {
                .hook                = ip_vs_in_hook,
                .pf                = NFPROTO_IPV6,
                .hooknum        = NF_INET_LOCAL_OUT,
                .priority        = NF_IP6_PRI_NAT_DST + 2,
        },
        /* After packet filtering (but before ip_vs_out_icmp), catch icmp
         * destined for 0.0.0.0/0, which is for incoming IPVS connections */
        {
                .hook                = ip_vs_forward_icmp,
                .pf                = NFPROTO_IPV6,
                .hooknum        = NF_INET_FORWARD,
                .priority        = 99,
        },
        /* After packet filtering, change source only for VS/NAT */
        {
                .hook                = ip_vs_out_hook,
                .pf                = NFPROTO_IPV6,
                .hooknum        = NF_INET_FORWARD,
                .priority        = 100,
        },
};
#endif

int ip_vs_register_hooks(struct netns_ipvs *ipvs, unsigned int af)
{
        const struct nf_hook_ops *ops;
        unsigned int count;
        unsigned int afmask;
        int ret = 0;

        if (af == AF_INET6) {
#ifdef CONFIG_IP_VS_IPV6
                ops = ip_vs_ops6;
                count = ARRAY_SIZE(ip_vs_ops6);
                afmask = 2;
#else
                return -EINVAL;
#endif
        } else {
                ops = ip_vs_ops4;
                count = ARRAY_SIZE(ip_vs_ops4);
                afmask = 1;
        }

        if (!(ipvs->hooks_afmask & afmask)) {
                ret = nf_register_net_hooks(ipvs->net, ops, count);
                if (ret >= 0)
                        ipvs->hooks_afmask |= afmask;
        }
        return ret;
}

void ip_vs_unregister_hooks(struct netns_ipvs *ipvs, unsigned int af)
{
        const struct nf_hook_ops *ops;
        unsigned int count;
        unsigned int afmask;

        if (af == AF_INET6) {
#ifdef CONFIG_IP_VS_IPV6
                ops = ip_vs_ops6;
                count = ARRAY_SIZE(ip_vs_ops6);
                afmask = 2;
#else
                return;
#endif
        } else {
                ops = ip_vs_ops4;
                count = ARRAY_SIZE(ip_vs_ops4);
                afmask = 1;
        }

        if (ipvs->hooks_afmask & afmask) {
                nf_unregister_net_hooks(ipvs->net, ops, count);
                ipvs->hooks_afmask &= ~afmask;
        }
}

/*
 *        Initialize IP Virtual Server netns mem.
 */
static int __net_init __ip_vs_init(struct net *net)
{
        struct netns_ipvs *ipvs;

        ipvs = net_generic(net, ip_vs_net_id);
        if (ipvs == NULL)
                return -ENOMEM;

        /* Hold the beast until a service is registered */
        ipvs->enable = 0;
        ipvs->net = net;
        /* Counters used for creating unique names */
        ipvs->gen = atomic_read(&ipvs_netns_cnt);
        atomic_inc(&ipvs_netns_cnt);
        net->ipvs = ipvs;

        if (ip_vs_estimator_net_init(ipvs) < 0)
                goto estimator_fail;

        if (ip_vs_control_net_init(ipvs) < 0)
                goto control_fail;

        if (ip_vs_protocol_net_init(ipvs) < 0)
                goto protocol_fail;

        if (ip_vs_app_net_init(ipvs) < 0)
                goto app_fail;

        if (ip_vs_conn_net_init(ipvs) < 0)
                goto conn_fail;

        if (ip_vs_sync_net_init(ipvs) < 0)
                goto sync_fail;

        return 0;
/*
 * Error handling
 */

sync_fail:
        ip_vs_conn_net_cleanup(ipvs);
conn_fail:
        ip_vs_app_net_cleanup(ipvs);
app_fail:
        ip_vs_protocol_net_cleanup(ipvs);
protocol_fail:
        ip_vs_control_net_cleanup(ipvs);
control_fail:
        ip_vs_estimator_net_cleanup(ipvs);
estimator_fail:
        net->ipvs = NULL;
        return -ENOMEM;
}

static void __net_exit __ip_vs_cleanup_batch(struct list_head *net_list)
{
        struct netns_ipvs *ipvs;
        struct net *net;

        ip_vs_service_nets_cleanup(net_list);        /* ip_vs_flush() with locks */
        list_for_each_entry(net, net_list, exit_list) {
                ipvs = net_ipvs(net);
                ip_vs_conn_net_cleanup(ipvs);
                ip_vs_app_net_cleanup(ipvs);
                ip_vs_protocol_net_cleanup(ipvs);
                ip_vs_control_net_cleanup(ipvs);
                ip_vs_estimator_net_cleanup(ipvs);
                IP_VS_DBG(2, "ipvs netns %d released\n", ipvs->gen);
                net->ipvs = NULL;
        }
}

static void __net_exit __ip_vs_dev_cleanup_batch(struct list_head *net_list)
{
        struct netns_ipvs *ipvs;
        struct net *net;

        list_for_each_entry(net, net_list, exit_list) {
                ipvs = net_ipvs(net);
                ip_vs_unregister_hooks(ipvs, AF_INET);
                ip_vs_unregister_hooks(ipvs, AF_INET6);
                ipvs->enable = 0;        /* Disable packet reception */
                smp_wmb();
                ip_vs_sync_net_cleanup(ipvs);
        }
}

static struct pernet_operations ipvs_core_ops = {
        .init = __ip_vs_init,
        .exit_batch = __ip_vs_cleanup_batch,
        .id   = &ip_vs_net_id,
        .size = sizeof(struct netns_ipvs),
};

static struct pernet_operations ipvs_core_dev_ops = {
        .exit_batch = __ip_vs_dev_cleanup_batch,
};

/*
 *        Initialize IP Virtual Server
 */
static int __init ip_vs_init(void)
{
        int ret;

        ret = ip_vs_control_init();
        if (ret < 0) {
                pr_err("can't setup control.\n");
                goto exit;
        }

        ip_vs_protocol_init();

        ret = ip_vs_conn_init();
        if (ret < 0) {
                pr_err("can't setup connection table.\n");
                goto cleanup_protocol;
        }

        ret = register_pernet_subsys(&ipvs_core_ops);        /* Alloc ip_vs struct */
        if (ret < 0)
                goto cleanup_conn;

        ret = register_pernet_device(&ipvs_core_dev_ops);
        if (ret < 0)
                goto cleanup_sub;

        ret = ip_vs_register_nl_ioctl();
        if (ret < 0) {
                pr_err("can't register netlink/ioctl.\n");
                goto cleanup_dev;
        }

        pr_info("ipvs loaded.\n");

        return ret;

cleanup_dev:
        unregister_pernet_device(&ipvs_core_dev_ops);
cleanup_sub:
        unregister_pernet_subsys(&ipvs_core_ops);
cleanup_conn:
        ip_vs_conn_cleanup();
cleanup_protocol:
        ip_vs_protocol_cleanup();
        ip_vs_control_cleanup();
exit:
        return ret;
}

static void __exit ip_vs_cleanup(void)
{
        ip_vs_unregister_nl_ioctl();
        unregister_pernet_device(&ipvs_core_dev_ops);
        unregister_pernet_subsys(&ipvs_core_ops);        /* free ip_vs struct */
        ip_vs_conn_cleanup();
        ip_vs_protocol_cleanup();
        ip_vs_control_cleanup();
        /* common rcu_barrier() used by:
         * - ip_vs_control_cleanup()
         */
        rcu_barrier();
        pr_info("ipvs unloaded.\n");
}

module_init(ip_vs_init);
module_exit(ip_vs_cleanup);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("IP Virtual Server");





















































































































































































































    4 





































    4 
    4 



















    4 































































































































    4 





























































































































































    4 
    4 



    4 



    4 












































    4 
    4 
    4 

    4 

    4 






    4 
    4 



    4 


    4 

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
// SPDX-License-Identifier: GPL-2.0-only
/*
 *        IEEE 802.1D Generic Attribute Registration Protocol (GARP)
 *
 *        Copyright (c) 2008 Patrick McHardy <kaber@trash.net>
 */
#include <linux/kernel.h>
#include <linux/timer.h>
#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/rtnetlink.h>
#include <linux/llc.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <net/llc.h>
#include <net/llc_pdu.h>
#include <net/garp.h>
#include <linux/unaligned.h>

static unsigned int garp_join_time __read_mostly = 200;
module_param(garp_join_time, uint, 0644);
MODULE_PARM_DESC(garp_join_time, "Join time in ms (default 200ms)");
MODULE_DESCRIPTION("IEEE 802.1D Generic Attribute Registration Protocol (GARP)");
MODULE_LICENSE("GPL");

static const struct garp_state_trans {
        u8        state;
        u8        action;
} garp_applicant_state_table[GARP_APPLICANT_MAX + 1][GARP_EVENT_MAX + 1] = {
        [GARP_APPLICANT_VA] = {
                [GARP_EVENT_TRANSMIT_PDU]        = { .state = GARP_APPLICANT_AA,
                                                    .action = GARP_ACTION_S_JOIN_IN },
                [GARP_EVENT_R_JOIN_IN]                = { .state = GARP_APPLICANT_AA },
                [GARP_EVENT_R_JOIN_EMPTY]        = { .state = GARP_APPLICANT_VA },
                [GARP_EVENT_R_EMPTY]                = { .state = GARP_APPLICANT_VA },
                [GARP_EVENT_R_LEAVE_IN]                = { .state = GARP_APPLICANT_VA },
                [GARP_EVENT_R_LEAVE_EMPTY]        = { .state = GARP_APPLICANT_VP },
                [GARP_EVENT_REQ_JOIN]                = { .state = GARP_APPLICANT_INVALID },
                [GARP_EVENT_REQ_LEAVE]                = { .state = GARP_APPLICANT_LA },
        },
        [GARP_APPLICANT_AA] = {
                [GARP_EVENT_TRANSMIT_PDU]        = { .state = GARP_APPLICANT_QA,
                                                    .action = GARP_ACTION_S_JOIN_IN },
                [GARP_EVENT_R_JOIN_IN]                = { .state = GARP_APPLICANT_QA },
                [GARP_EVENT_R_JOIN_EMPTY]        = { .state = GARP_APPLICANT_VA },
                [GARP_EVENT_R_EMPTY]                = { .state = GARP_APPLICANT_VA },
                [GARP_EVENT_R_LEAVE_IN]                = { .state = GARP_APPLICANT_VA },
                [GARP_EVENT_R_LEAVE_EMPTY]        = { .state = GARP_APPLICANT_VP },
                [GARP_EVENT_REQ_JOIN]                = { .state = GARP_APPLICANT_INVALID },
                [GARP_EVENT_REQ_LEAVE]                = { .state = GARP_APPLICANT_LA },
        },
        [GARP_APPLICANT_QA] = {
                [GARP_EVENT_TRANSMIT_PDU]        = { .state = GARP_APPLICANT_INVALID },
                [GARP_EVENT_R_JOIN_IN]                = { .state = GARP_APPLICANT_QA },
                [GARP_EVENT_R_JOIN_EMPTY]        = { .state = GARP_APPLICANT_VA },
                [GARP_EVENT_R_EMPTY]                = { .state = GARP_APPLICANT_VA },
                [GARP_EVENT_R_LEAVE_IN]                = { .state = GARP_APPLICANT_VP },
                [GARP_EVENT_R_LEAVE_EMPTY]        = { .state = GARP_APPLICANT_VP },
                [GARP_EVENT_REQ_JOIN]                = { .state = GARP_APPLICANT_INVALID },
                [GARP_EVENT_REQ_LEAVE]                = { .state = GARP_APPLICANT_LA },
        },
        [GARP_APPLICANT_LA] = {
                [GARP_EVENT_TRANSMIT_PDU]        = { .state = GARP_APPLICANT_VO,
                                                    .action = GARP_ACTION_S_LEAVE_EMPTY },
                [GARP_EVENT_R_JOIN_IN]                = { .state = GARP_APPLICANT_LA },
                [GARP_EVENT_R_JOIN_EMPTY]        = { .state = GARP_APPLICANT_VO },
                [GARP_EVENT_R_EMPTY]                = { .state = GARP_APPLICANT_LA },
                [GARP_EVENT_R_LEAVE_IN]                = { .state = GARP_APPLICANT_LA },
                [GARP_EVENT_R_LEAVE_EMPTY]        = { .state = GARP_APPLICANT_VO },
                [GARP_EVENT_REQ_JOIN]                = { .state = GARP_APPLICANT_VA },
                [GARP_EVENT_REQ_LEAVE]                = { .state = GARP_APPLICANT_INVALID },
        },
        [GARP_APPLICANT_VP] = {
                [GARP_EVENT_TRANSMIT_PDU]        = { .state = GARP_APPLICANT_AA,
                                                    .action = GARP_ACTION_S_JOIN_IN },
                [GARP_EVENT_R_JOIN_IN]                = { .state = GARP_APPLICANT_AP },
                [GARP_EVENT_R_JOIN_EMPTY]        = { .state = GARP_APPLICANT_VP },
                [GARP_EVENT_R_EMPTY]                = { .state = GARP_APPLICANT_VP },
                [GARP_EVENT_R_LEAVE_IN]                = { .state = GARP_APPLICANT_VP },
                [GARP_EVENT_R_LEAVE_EMPTY]        = { .state = GARP_APPLICANT_VP },
                [GARP_EVENT_REQ_JOIN]                = { .state = GARP_APPLICANT_INVALID },
                [GARP_EVENT_REQ_LEAVE]                = { .state = GARP_APPLICANT_VO },
        },
        [GARP_APPLICANT_AP] = {
                [GARP_EVENT_TRANSMIT_PDU]        = { .state = GARP_APPLICANT_QA,
                                                    .action = GARP_ACTION_S_JOIN_IN },
                [GARP_EVENT_R_JOIN_IN]                = { .state = GARP_APPLICANT_QP },
                [GARP_EVENT_R_JOIN_EMPTY]        = { .state = GARP_APPLICANT_VP },
                [GARP_EVENT_R_EMPTY]                = { .state = GARP_APPLICANT_VP },
                [GARP_EVENT_R_LEAVE_IN]                = { .state = GARP_APPLICANT_VP },
                [GARP_EVENT_R_LEAVE_EMPTY]        = { .state = GARP_APPLICANT_VP },
                [GARP_EVENT_REQ_JOIN]                = { .state = GARP_APPLICANT_INVALID },
                [GARP_EVENT_REQ_LEAVE]                = { .state = GARP_APPLICANT_AO },
        },
        [GARP_APPLICANT_QP] = {
                [GARP_EVENT_TRANSMIT_PDU]        = { .state = GARP_APPLICANT_INVALID },
                [GARP_EVENT_R_JOIN_IN]                = { .state = GARP_APPLICANT_QP },
                [GARP_EVENT_R_JOIN_EMPTY]        = { .state = GARP_APPLICANT_VP },
                [GARP_EVENT_R_EMPTY]                = { .state = GARP_APPLICANT_VP },
                [GARP_EVENT_R_LEAVE_IN]                = { .state = GARP_APPLICANT_VP },
                [GARP_EVENT_R_LEAVE_EMPTY]        = { .state = GARP_APPLICANT_VP },
                [GARP_EVENT_REQ_JOIN]                = { .state = GARP_APPLICANT_INVALID },
                [GARP_EVENT_REQ_LEAVE]                = { .state = GARP_APPLICANT_QO },
        },
        [GARP_APPLICANT_VO] = {
                [GARP_EVENT_TRANSMIT_PDU]        = { .state = GARP_APPLICANT_INVALID },
                [GARP_EVENT_R_JOIN_IN]                = { .state = GARP_APPLICANT_AO },
                [GARP_EVENT_R_JOIN_EMPTY]        = { .state = GARP_APPLICANT_VO },
                [GARP_EVENT_R_EMPTY]                = { .state = GARP_APPLICANT_VO },
                [GARP_EVENT_R_LEAVE_IN]                = { .state = GARP_APPLICANT_VO },
                [GARP_EVENT_R_LEAVE_EMPTY]        = { .state = GARP_APPLICANT_VO },
                [GARP_EVENT_REQ_JOIN]                = { .state = GARP_APPLICANT_VP },
                [GARP_EVENT_REQ_LEAVE]                = { .state = GARP_APPLICANT_INVALID },
        },
        [GARP_APPLICANT_AO] = {
                [GARP_EVENT_TRANSMIT_PDU]        = { .state = GARP_APPLICANT_INVALID },
                [GARP_EVENT_R_JOIN_IN]                = { .state = GARP_APPLICANT_QO },
                [GARP_EVENT_R_JOIN_EMPTY]        = { .state = GARP_APPLICANT_VO },
                [GARP_EVENT_R_EMPTY]                = { .state = GARP_APPLICANT_VO },
                [GARP_EVENT_R_LEAVE_IN]                = { .state = GARP_APPLICANT_VO },
                [GARP_EVENT_R_LEAVE_EMPTY]        = { .state = GARP_APPLICANT_VO },
                [GARP_EVENT_REQ_JOIN]                = { .state = GARP_APPLICANT_AP },
                [GARP_EVENT_REQ_LEAVE]                = { .state = GARP_APPLICANT_INVALID },
        },
        [GARP_APPLICANT_QO] = {
                [GARP_EVENT_TRANSMIT_PDU]        = { .state = GARP_APPLICANT_INVALID },
                [GARP_EVENT_R_JOIN_IN]                = { .state = GARP_APPLICANT_QO },
                [GARP_EVENT_R_JOIN_EMPTY]        = { .state = GARP_APPLICANT_VO },
                [GARP_EVENT_R_EMPTY]                = { .state = GARP_APPLICANT_VO },
                [GARP_EVENT_R_LEAVE_IN]                = { .state = GARP_APPLICANT_VO },
                [GARP_EVENT_R_LEAVE_EMPTY]        = { .state = GARP_APPLICANT_VO },
                [GARP_EVENT_REQ_JOIN]                = { .state = GARP_APPLICANT_QP },
                [GARP_EVENT_REQ_LEAVE]                = { .state = GARP_APPLICANT_INVALID },
        },
};

static int garp_attr_cmp(const struct garp_attr *attr,
                         const void *data, u8 len, u8 type)
{
        if (attr->type != type)
                return attr->type - type;
        if (attr->dlen != len)
                return attr->dlen - len;
        return memcmp(attr->data, data, len);
}

static struct garp_attr *garp_attr_lookup(const struct garp_applicant *app,
                                          const void *data, u8 len, u8 type)
{
        struct rb_node *parent = app->gid.rb_node;
        struct garp_attr *attr;
        int d;

        while (parent) {
                attr = rb_entry(parent, struct garp_attr, node);
                d = garp_attr_cmp(attr, data, len, type);
                if (d > 0)
                        parent = parent->rb_left;
                else if (d < 0)
                        parent = parent->rb_right;
                else
                        return attr;
        }
        return NULL;
}

static struct garp_attr *garp_attr_create(struct garp_applicant *app,
                                          const void *data, u8 len, u8 type)
{
        struct rb_node *parent = NULL, **p = &app->gid.rb_node;
        struct garp_attr *attr;
        int d;

        while (*p) {
                parent = *p;
                attr = rb_entry(parent, struct garp_attr, node);
                d = garp_attr_cmp(attr, data, len, type);
                if (d > 0)
                        p = &parent->rb_left;
                else if (d < 0)
                        p = &parent->rb_right;
                else {
                        /* The attribute already exists; re-use it. */
                        return attr;
                }
        }
        attr = kmalloc(sizeof(*attr) + len, GFP_ATOMIC);
        if (!attr)
                return attr;
        attr->state = GARP_APPLICANT_VO;
        attr->type  = type;
        attr->dlen  = len;
        memcpy(attr->data, data, len);

        rb_link_node(&attr->node, parent, p);
        rb_insert_color(&attr->node, &app->gid);
        return attr;
}

static void garp_attr_destroy(struct garp_applicant *app, struct garp_attr *attr)
{
        rb_erase(&attr->node, &app->gid);
        kfree(attr);
}

static void garp_attr_destroy_all(struct garp_applicant *app)
{
        struct rb_node *node, *next;
        struct garp_attr *attr;

        for (node = rb_first(&app->gid);
             next = node ? rb_next(node) : NULL, node != NULL;
             node = next) {
                attr = rb_entry(node, struct garp_attr, node);
                garp_attr_destroy(app, attr);
        }
}

static int garp_pdu_init(struct garp_applicant *app)
{
        struct sk_buff *skb;
        struct garp_pdu_hdr *gp;

#define LLC_RESERVE        sizeof(struct llc_pdu_un)
        skb = alloc_skb(app->dev->mtu + LL_RESERVED_SPACE(app->dev),
                        GFP_ATOMIC);
        if (!skb)
                return -ENOMEM;

        skb->dev = app->dev;
        skb->protocol = htons(ETH_P_802_2);
        skb_reserve(skb, LL_RESERVED_SPACE(app->dev) + LLC_RESERVE);

        gp = __skb_put(skb, sizeof(*gp));
        put_unaligned(htons(GARP_PROTOCOL_ID), &gp->protocol);

        app->pdu = skb;
        return 0;
}

static int garp_pdu_append_end_mark(struct garp_applicant *app)
{
        if (skb_tailroom(app->pdu) < sizeof(u8))
                return -1;
        __skb_put_u8(app->pdu, GARP_END_MARK);
        return 0;
}

static void garp_pdu_queue(struct garp_applicant *app)
{
        if (!app->pdu)
                return;

        garp_pdu_append_end_mark(app);
        garp_pdu_append_end_mark(app);

        llc_pdu_header_init(app->pdu, LLC_PDU_TYPE_U, LLC_SAP_BSPAN,
                            LLC_SAP_BSPAN, LLC_PDU_CMD);
        llc_pdu_init_as_ui_cmd(app->pdu);
        llc_mac_hdr_init(app->pdu, app->dev->dev_addr,
                         app->app->proto.group_address);

        skb_queue_tail(&app->queue, app->pdu);
        app->pdu = NULL;
}

static void garp_queue_xmit(struct garp_applicant *app)
{
        struct sk_buff *skb;

        while ((skb = skb_dequeue(&app->queue)))
                dev_queue_xmit(skb);
}

static int garp_pdu_append_msg(struct garp_applicant *app, u8 attrtype)
{
        struct garp_msg_hdr *gm;

        if (skb_tailroom(app->pdu) < sizeof(*gm))
                return -1;
        gm = __skb_put(app->pdu, sizeof(*gm));
        gm->attrtype = attrtype;
        garp_cb(app->pdu)->cur_type = attrtype;
        return 0;
}

static int garp_pdu_append_attr(struct garp_applicant *app,
                                const struct garp_attr *attr,
                                enum garp_attr_event event)
{
        struct garp_attr_hdr *ga;
        unsigned int len;
        int err;
again:
        if (!app->pdu) {
                err = garp_pdu_init(app);
                if (err < 0)
                        return err;
        }

        if (garp_cb(app->pdu)->cur_type != attr->type) {
                if (garp_cb(app->pdu)->cur_type &&
                    garp_pdu_append_end_mark(app) < 0)
                        goto queue;
                if (garp_pdu_append_msg(app, attr->type) < 0)
                        goto queue;
        }

        len = sizeof(*ga) + attr->dlen;
        if (skb_tailroom(app->pdu) < len)
                goto queue;
        ga = __skb_put(app->pdu, len);
        ga->len   = len;
        ga->event = event;
        memcpy(ga->data, attr->data, attr->dlen);
        return 0;

queue:
        garp_pdu_queue(app);
        goto again;
}

static void garp_attr_event(struct garp_applicant *app,
                            struct garp_attr *attr, enum garp_event event)
{
        enum garp_applicant_state state;

        state = garp_applicant_state_table[attr->state][event].state;
        if (state == GARP_APPLICANT_INVALID)
                return;

        switch (garp_applicant_state_table[attr->state][event].action) {
        case GARP_ACTION_NONE:
                break;
        case GARP_ACTION_S_JOIN_IN:
                /* When appending the attribute fails, don't update state in
                 * order to retry on next TRANSMIT_PDU event. */
                if (garp_pdu_append_attr(app, attr, GARP_JOIN_IN) < 0)
                        return;
                break;
        case GARP_ACTION_S_LEAVE_EMPTY:
                garp_pdu_append_attr(app, attr, GARP_LEAVE_EMPTY);
                /* As a pure applicant, sending a leave message implies that
                 * the attribute was unregistered and can be destroyed. */
                garp_attr_destroy(app, attr);
                return;
        default:
                WARN_ON(1);
        }

        attr->state = state;
}

int garp_request_join(const struct net_device *dev,
                      const struct garp_application *appl,
                      const void *data, u8 len, u8 type)
{
        struct garp_port *port = rtnl_dereference(dev->garp_port);
        struct garp_applicant *app = rtnl_dereference(port->applicants[appl->type]);
        struct garp_attr *attr;

        spin_lock_bh(&app->lock);
        attr = garp_attr_create(app, data, len, type);
        if (!attr) {
                spin_unlock_bh(&app->lock);
                return -ENOMEM;
        }
        garp_attr_event(app, attr, GARP_EVENT_REQ_JOIN);
        spin_unlock_bh(&app->lock);
        return 0;
}
EXPORT_SYMBOL_GPL(garp_request_join);

void garp_request_leave(const struct net_device *dev,
                        const struct garp_application *appl,
                        const void *data, u8 len, u8 type)
{
        struct garp_port *port = rtnl_dereference(dev->garp_port);
        struct garp_applicant *app = rtnl_dereference(port->applicants[appl->type]);
        struct garp_attr *attr;

        spin_lock_bh(&app->lock);
        attr = garp_attr_lookup(app, data, len, type);
        if (!attr) {
                spin_unlock_bh(&app->lock);
                return;
        }
        garp_attr_event(app, attr, GARP_EVENT_REQ_LEAVE);
        spin_unlock_bh(&app->lock);
}
EXPORT_SYMBOL_GPL(garp_request_leave);

static void garp_gid_event(struct garp_applicant *app, enum garp_event event)
{
        struct rb_node *node, *next;
        struct garp_attr *attr;

        for (node = rb_first(&app->gid);
             next = node ? rb_next(node) : NULL, node != NULL;
             node = next) {
                attr = rb_entry(node, struct garp_attr, node);
                garp_attr_event(app, attr, event);
        }
}

static void garp_join_timer_arm(struct garp_applicant *app)
{
        unsigned long delay;

        delay = get_random_u32_below(msecs_to_jiffies(garp_join_time));
        mod_timer(&app->join_timer, jiffies + delay);
}

static void garp_join_timer(struct timer_list *t)
{
        struct garp_applicant *app = from_timer(app, t, join_timer);

        spin_lock(&app->lock);
        garp_gid_event(app, GARP_EVENT_TRANSMIT_PDU);
        garp_pdu_queue(app);
        spin_unlock(&app->lock);

        garp_queue_xmit(app);
        garp_join_timer_arm(app);
}

static int garp_pdu_parse_end_mark(struct sk_buff *skb)
{
        if (!pskb_may_pull(skb, sizeof(u8)))
                return -1;
        if (*skb->data == GARP_END_MARK) {
                skb_pull(skb, sizeof(u8));
                return -1;
        }
        return 0;
}

static int garp_pdu_parse_attr(struct garp_applicant *app, struct sk_buff *skb,
                               u8 attrtype)
{
        const struct garp_attr_hdr *ga;
        struct garp_attr *attr;
        enum garp_event event;
        unsigned int dlen;

        if (!pskb_may_pull(skb, sizeof(*ga)))
                return -1;
        ga = (struct garp_attr_hdr *)skb->data;
        if (ga->len < sizeof(*ga))
                return -1;

        if (!pskb_may_pull(skb, ga->len))
                return -1;
        skb_pull(skb, ga->len);
        dlen = sizeof(*ga) - ga->len;

        if (attrtype > app->app->maxattr)
                return 0;

        switch (ga->event) {
        case GARP_LEAVE_ALL:
                if (dlen != 0)
                        return -1;
                garp_gid_event(app, GARP_EVENT_R_LEAVE_EMPTY);
                return 0;
        case GARP_JOIN_EMPTY:
                event = GARP_EVENT_R_JOIN_EMPTY;
                break;
        case GARP_JOIN_IN:
                event = GARP_EVENT_R_JOIN_IN;
                break;
        case GARP_LEAVE_EMPTY:
                event = GARP_EVENT_R_LEAVE_EMPTY;
                break;
        case GARP_EMPTY:
                event = GARP_EVENT_R_EMPTY;
                break;
        default:
                return 0;
        }

        if (dlen == 0)
                return -1;
        attr = garp_attr_lookup(app, ga->data, dlen, attrtype);
        if (attr == NULL)
                return 0;
        garp_attr_event(app, attr, event);
        return 0;
}

static int garp_pdu_parse_msg(struct garp_applicant *app, struct sk_buff *skb)
{
        const struct garp_msg_hdr *gm;

        if (!pskb_may_pull(skb, sizeof(*gm)))
                return -1;
        gm = (struct garp_msg_hdr *)skb->data;
        if (gm->attrtype == 0)
                return -1;
        skb_pull(skb, sizeof(*gm));

        while (skb->len > 0) {
                if (garp_pdu_parse_attr(app, skb, gm->attrtype) < 0)
                        return -1;
                if (garp_pdu_parse_end_mark(skb) < 0)
                        break;
        }
        return 0;
}

static void garp_pdu_rcv(const struct stp_proto *proto, struct sk_buff *skb,
                         struct net_device *dev)
{
        struct garp_application *appl = proto->data;
        struct garp_port *port;
        struct garp_applicant *app;
        const struct garp_pdu_hdr *gp;

        port = rcu_dereference(dev->garp_port);
        if (!port)
                goto err;
        app = rcu_dereference(port->applicants[appl->type]);
        if (!app)
                goto err;

        if (!pskb_may_pull(skb, sizeof(*gp)))
                goto err;
        gp = (struct garp_pdu_hdr *)skb->data;
        if (get_unaligned(&gp->protocol) != htons(GARP_PROTOCOL_ID))
                goto err;
        skb_pull(skb, sizeof(*gp));

        spin_lock(&app->lock);
        while (skb->len > 0) {
                if (garp_pdu_parse_msg(app, skb) < 0)
                        break;
                if (garp_pdu_parse_end_mark(skb) < 0)
                        break;
        }
        spin_unlock(&app->lock);
err:
        kfree_skb(skb);
}

static int garp_init_port(struct net_device *dev)
{
        struct garp_port *port;

        port = kzalloc(sizeof(*port), GFP_KERNEL);
        if (!port)
                return -ENOMEM;
        rcu_assign_pointer(dev->garp_port, port);
        return 0;
}

static void garp_release_port(struct net_device *dev)
{
        struct garp_port *port = rtnl_dereference(dev->garp_port);
        unsigned int i;

        for (i = 0; i <= GARP_APPLICATION_MAX; i++) {
                if (rtnl_dereference(port->applicants[i]))
                        return;
        }
        RCU_INIT_POINTER(dev->garp_port, NULL);
        kfree_rcu(port, rcu);
}

int garp_init_applicant(struct net_device *dev, struct garp_application *appl)
{
        struct garp_applicant *app;
        int err;

        ASSERT_RTNL();

        if (!rtnl_dereference(dev->garp_port)) {
                err = garp_init_port(dev);
                if (err < 0)
                        goto err1;
        }

        err = -ENOMEM;
        app = kzalloc(sizeof(*app), GFP_KERNEL);
        if (!app)
                goto err2;

        err = dev_mc_add(dev, appl->proto.group_address);
        if (err < 0)
                goto err3;

        app->dev = dev;
        app->app = appl;
        app->gid = RB_ROOT;
        spin_lock_init(&app->lock);
        skb_queue_head_init(&app->queue);
        rcu_assign_pointer(dev->garp_port->applicants[appl->type], app);
        timer_setup(&app->join_timer, garp_join_timer, 0);
        garp_join_timer_arm(app);
        return 0;

err3:
        kfree(app);
err2:
        garp_release_port(dev);
err1:
        return err;
}
EXPORT_SYMBOL_GPL(garp_init_applicant);

void garp_uninit_applicant(struct net_device *dev, struct garp_application *appl)
{
        struct garp_port *port = rtnl_dereference(dev->garp_port);
        struct garp_applicant *app = rtnl_dereference(port->applicants[appl->type]);

        ASSERT_RTNL();

        RCU_INIT_POINTER(port->applicants[appl->type], NULL);

        /* Delete timer and generate a final TRANSMIT_PDU event to flush out
         * all pending messages before the applicant is gone. */
        timer_shutdown_sync(&app->join_timer);

        spin_lock_bh(&app->lock);
        garp_gid_event(app, GARP_EVENT_TRANSMIT_PDU);
        garp_attr_destroy_all(app);
        garp_pdu_queue(app);
        spin_unlock_bh(&app->lock);

        garp_queue_xmit(app);

        dev_mc_del(dev, appl->proto.group_address);
        kfree_rcu(app, rcu);
        garp_release_port(dev);
}
EXPORT_SYMBOL_GPL(garp_uninit_applicant);

int garp_register_application(struct garp_application *appl)
{
        appl->proto.rcv = garp_pdu_rcv;
        appl->proto.data = appl;
        return stp_proto_register(&appl->proto);
}
EXPORT_SYMBOL_GPL(garp_register_application);

void garp_unregister_application(struct garp_application *appl)
{
        stp_proto_unregister(&appl->proto);
}
EXPORT_SYMBOL_GPL(garp_unregister_application);






































































































































































































































































































    7 
    7 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * bvec iterator
 *
 * Copyright (C) 2001 Ming Lei <ming.lei@canonical.com>
 */
#ifndef __LINUX_BVEC_H
#define __LINUX_BVEC_H

#include <linux/highmem.h>
#include <linux/bug.h>
#include <linux/errno.h>
#include <linux/limits.h>
#include <linux/minmax.h>
#include <linux/types.h>

struct page;

/**
 * struct bio_vec - a contiguous range of physical memory addresses
 * @bv_page:   First page associated with the address range.
 * @bv_len:    Number of bytes in the address range.
 * @bv_offset: Start of the address range relative to the start of @bv_page.
 *
 * The following holds for a bvec if n * PAGE_SIZE < bv_offset + bv_len:
 *
 *   nth_page(@bv_page, n) == @bv_page + n
 *
 * This holds because page_is_mergeable() checks the above property.
 */
struct bio_vec {
        struct page        *bv_page;
        unsigned int        bv_len;
        unsigned int        bv_offset;
};

/**
 * bvec_set_page - initialize a bvec based off a struct page
 * @bv:                bvec to initialize
 * @page:        page the bvec should point to
 * @len:        length of the bvec
 * @offset:        offset into the page
 */
static inline void bvec_set_page(struct bio_vec *bv, struct page *page,
                unsigned int len, unsigned int offset)
{
        bv->bv_page = page;
        bv->bv_len = len;
        bv->bv_offset = offset;
}

/**
 * bvec_set_folio - initialize a bvec based off a struct folio
 * @bv:                bvec to initialize
 * @folio:        folio the bvec should point to
 * @len:        length of the bvec
 * @offset:        offset into the folio
 */
static inline void bvec_set_folio(struct bio_vec *bv, struct folio *folio,
                unsigned int len, unsigned int offset)
{
        bvec_set_page(bv, &folio->page, len, offset);
}

/**
 * bvec_set_virt - initialize a bvec based on a virtual address
 * @bv:                bvec to initialize
 * @vaddr:        virtual address to set the bvec to
 * @len:        length of the bvec
 */
static inline void bvec_set_virt(struct bio_vec *bv, void *vaddr,
                unsigned int len)
{
        bvec_set_page(bv, virt_to_page(vaddr), len, offset_in_page(vaddr));
}

struct bvec_iter {
        sector_t                bi_sector;        /* device address in 512 byte
                                                   sectors */
        unsigned int                bi_size;        /* residual I/O count */

        unsigned int                bi_idx;                /* current index into bvl_vec */

        unsigned int            bi_bvec_done;        /* number of bytes completed in
                                                   current bvec */
} __packed __aligned(4);

struct bvec_iter_all {
        struct bio_vec        bv;
        int                idx;
        unsigned        done;
};

/*
 * various member access, note that bio_data should of course not be used
 * on highmem page vectors
 */
#define __bvec_iter_bvec(bvec, iter)        (&(bvec)[(iter).bi_idx])

/* multi-page (mp_bvec) helpers */
#define mp_bvec_iter_page(bvec, iter)                                \
        (__bvec_iter_bvec((bvec), (iter))->bv_page)

#define mp_bvec_iter_len(bvec, iter)                                \
        min((iter).bi_size,                                        \
            __bvec_iter_bvec((bvec), (iter))->bv_len - (iter).bi_bvec_done)

#define mp_bvec_iter_offset(bvec, iter)                                \
        (__bvec_iter_bvec((bvec), (iter))->bv_offset + (iter).bi_bvec_done)

#define mp_bvec_iter_page_idx(bvec, iter)                        \
        (mp_bvec_iter_offset((bvec), (iter)) / PAGE_SIZE)

#define mp_bvec_iter_bvec(bvec, iter)                                \
((struct bio_vec) {                                                \
        .bv_page        = mp_bvec_iter_page((bvec), (iter)),        \
        .bv_len                = mp_bvec_iter_len((bvec), (iter)),        \
        .bv_offset        = mp_bvec_iter_offset((bvec), (iter)),        \
})

/* For building single-page bvec in flight */
 #define bvec_iter_offset(bvec, iter)                                \
        (mp_bvec_iter_offset((bvec), (iter)) % PAGE_SIZE)

#define bvec_iter_len(bvec, iter)                                \
        min_t(unsigned, mp_bvec_iter_len((bvec), (iter)),                \
              PAGE_SIZE - bvec_iter_offset((bvec), (iter)))

#define bvec_iter_page(bvec, iter)                                \
        (mp_bvec_iter_page((bvec), (iter)) +                        \
         mp_bvec_iter_page_idx((bvec), (iter)))

#define bvec_iter_bvec(bvec, iter)                                \
((struct bio_vec) {                                                \
        .bv_page        = bvec_iter_page((bvec), (iter)),        \
        .bv_len                = bvec_iter_len((bvec), (iter)),        \
        .bv_offset        = bvec_iter_offset((bvec), (iter)),        \
})

static inline bool bvec_iter_advance(const struct bio_vec *bv,
                struct bvec_iter *iter, unsigned bytes)
{
        unsigned int idx = iter->bi_idx;

        if (WARN_ONCE(bytes > iter->bi_size,
                     "Attempted to advance past end of bvec iter\n")) {
                iter->bi_size = 0;
                return false;
        }

        iter->bi_size -= bytes;
        bytes += iter->bi_bvec_done;

        while (bytes && bytes >= bv[idx].bv_len) {
                bytes -= bv[idx].bv_len;
                idx++;
        }

        iter->bi_idx = idx;
        iter->bi_bvec_done = bytes;
        return true;
}

/*
 * A simpler version of bvec_iter_advance(), @bytes should not span
 * across multiple bvec entries, i.e. bytes <= bv[i->bi_idx].bv_len
 */
static inline void bvec_iter_advance_single(const struct bio_vec *bv,
                                struct bvec_iter *iter, unsigned int bytes)
{
        unsigned int done = iter->bi_bvec_done + bytes;

        if (done == bv[iter->bi_idx].bv_len) {
                done = 0;
                iter->bi_idx++;
        }
        iter->bi_bvec_done = done;
        iter->bi_size -= bytes;
}

#define for_each_bvec(bvl, bio_vec, iter, start)                        \
        for (iter = (start);                                                \
             (iter).bi_size &&                                                \
                ((bvl = bvec_iter_bvec((bio_vec), (iter))), 1);        \
             bvec_iter_advance_single((bio_vec), &(iter), (bvl).bv_len))

#define for_each_mp_bvec(bvl, bio_vec, iter, start)                        \
        for (iter = (start);                                                \
             (iter).bi_size &&                                                \
                ((bvl = mp_bvec_iter_bvec((bio_vec), (iter))), 1);        \
             bvec_iter_advance_single((bio_vec), &(iter), (bvl).bv_len))

/* for iterating one bio from start to end */
#define BVEC_ITER_ALL_INIT (struct bvec_iter)                                \
{                                                                        \
        .bi_sector        = 0,                                                \
        .bi_size        = UINT_MAX,                                        \
        .bi_idx                = 0,                                                \
        .bi_bvec_done        = 0,                                                \
}

static inline struct bio_vec *bvec_init_iter_all(struct bvec_iter_all *iter_all)
{
        iter_all->done = 0;
        iter_all->idx = 0;

        return &iter_all->bv;
}

static inline void bvec_advance(const struct bio_vec *bvec,
                                struct bvec_iter_all *iter_all)
{
        struct bio_vec *bv = &iter_all->bv;

        if (iter_all->done) {
                bv->bv_page++;
                bv->bv_offset = 0;
        } else {
                bv->bv_page = bvec->bv_page + (bvec->bv_offset >> PAGE_SHIFT);
                bv->bv_offset = bvec->bv_offset & ~PAGE_MASK;
        }
        bv->bv_len = min_t(unsigned int, PAGE_SIZE - bv->bv_offset,
                           bvec->bv_len - iter_all->done);
        iter_all->done += bv->bv_len;

        if (iter_all->done == bvec->bv_len) {
                iter_all->idx++;
                iter_all->done = 0;
        }
}

/**
 * bvec_kmap_local - map a bvec into the kernel virtual address space
 * @bvec: bvec to map
 *
 * Must be called on single-page bvecs only.  Call kunmap_local on the returned
 * address to unmap.
 */
static inline void *bvec_kmap_local(struct bio_vec *bvec)
{
        return kmap_local_page(bvec->bv_page) + bvec->bv_offset;
}

/**
 * memcpy_from_bvec - copy data from a bvec
 * @bvec: bvec to copy from
 *
 * Must be called on single-page bvecs only.
 */
static inline void memcpy_from_bvec(char *to, struct bio_vec *bvec)
{
        memcpy_from_page(to, bvec->bv_page, bvec->bv_offset, bvec->bv_len);
}

/**
 * memcpy_to_bvec - copy data to a bvec
 * @bvec: bvec to copy to
 *
 * Must be called on single-page bvecs only.
 */
static inline void memcpy_to_bvec(struct bio_vec *bvec, const char *from)
{
        memcpy_to_page(bvec->bv_page, bvec->bv_offset, from, bvec->bv_len);
}

/**
 * memzero_bvec - zero all data in a bvec
 * @bvec: bvec to zero
 *
 * Must be called on single-page bvecs only.
 */
static inline void memzero_bvec(struct bio_vec *bvec)
{
        memzero_page(bvec->bv_page, bvec->bv_offset, bvec->bv_len);
}

/**
 * bvec_virt - return the virtual address for a bvec
 * @bvec: bvec to return the virtual address for
 *
 * Note: the caller must ensure that @bvec->bv_page is not a highmem page.
 */
static inline void *bvec_virt(struct bio_vec *bvec)
{
        WARN_ON_ONCE(PageHighMem(bvec->bv_page));
        return page_address(bvec->bv_page) + bvec->bv_offset;
}

/**
 * bvec_phys - return the physical address for a bvec
 * @bvec: bvec to return the physical address for
 */
static inline phys_addr_t bvec_phys(const struct bio_vec *bvec)
{
        return page_to_phys(bvec->bv_page) + bvec->bv_offset;
}

#endif /* __LINUX_BVEC_H */













 5590 



 5591 



 5584 
 5590 

 5582 
























 3773 
 3770 





























































    9 

















    6 

    1 














































































   10 


   15 



   11 

    5 






    5 




























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
#include <linux/bpf.h>
#include <linux/vmalloc.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/kernel.h>
#include <linux/idr.h>
#include <linux/namei.h>
#include <linux/user_namespace.h>
#include <linux/security.h>

static bool bpf_ns_capable(struct user_namespace *ns, int cap)
{
        return ns_capable(ns, cap) || (cap != CAP_SYS_ADMIN && ns_capable(ns, CAP_SYS_ADMIN));
}

bool bpf_token_capable(const struct bpf_token *token, int cap)
{
        struct user_namespace *userns;

        /* BPF token allows ns_capable() level of capabilities */
        userns = token ? token->userns : &init_user_ns;
        if (!bpf_ns_capable(userns, cap))
                return false;
        if (token && security_bpf_token_capable(token, cap) < 0)
                return false;
        return true;
}

void bpf_token_inc(struct bpf_token *token)
{
        atomic64_inc(&token->refcnt);
}

static void bpf_token_free(struct bpf_token *token)
{
        security_bpf_token_free(token);
        put_user_ns(token->userns);
        kfree(token);
}

static void bpf_token_put_deferred(struct work_struct *work)
{
        struct bpf_token *token = container_of(work, struct bpf_token, work);

        bpf_token_free(token);
}

void bpf_token_put(struct bpf_token *token)
{
        if (!token)
                return;

        if (!atomic64_dec_and_test(&token->refcnt))
                return;

        INIT_WORK(&token->work, bpf_token_put_deferred);
        schedule_work(&token->work);
}

static int bpf_token_release(struct inode *inode, struct file *filp)
{
        struct bpf_token *token = filp->private_data;

        bpf_token_put(token);
        return 0;
}

static void bpf_token_show_fdinfo(struct seq_file *m, struct file *filp)
{
        struct bpf_token *token = filp->private_data;
        u64 mask;

        BUILD_BUG_ON(__MAX_BPF_CMD >= 64);
        mask = BIT_ULL(__MAX_BPF_CMD) - 1;
        if ((token->allowed_cmds & mask) == mask)
                seq_printf(m, "allowed_cmds:\tany\n");
        else
                seq_printf(m, "allowed_cmds:\t0x%llx\n", token->allowed_cmds);

        BUILD_BUG_ON(__MAX_BPF_MAP_TYPE >= 64);
        mask = BIT_ULL(__MAX_BPF_MAP_TYPE) - 1;
        if ((token->allowed_maps & mask) == mask)
                seq_printf(m, "allowed_maps:\tany\n");
        else
                seq_printf(m, "allowed_maps:\t0x%llx\n", token->allowed_maps);

        BUILD_BUG_ON(__MAX_BPF_PROG_TYPE >= 64);
        mask = BIT_ULL(__MAX_BPF_PROG_TYPE) - 1;
        if ((token->allowed_progs & mask) == mask)
                seq_printf(m, "allowed_progs:\tany\n");
        else
                seq_printf(m, "allowed_progs:\t0x%llx\n", token->allowed_progs);

        BUILD_BUG_ON(__MAX_BPF_ATTACH_TYPE >= 64);
        mask = BIT_ULL(__MAX_BPF_ATTACH_TYPE) - 1;
        if ((token->allowed_attachs & mask) == mask)
                seq_printf(m, "allowed_attachs:\tany\n");
        else
                seq_printf(m, "allowed_attachs:\t0x%llx\n", token->allowed_attachs);
}

#define BPF_TOKEN_INODE_NAME "bpf-token"

static const struct inode_operations bpf_token_iops = { };

static const struct file_operations bpf_token_fops = {
        .release        = bpf_token_release,
        .show_fdinfo        = bpf_token_show_fdinfo,
};

int bpf_token_create(union bpf_attr *attr)
{
        struct bpf_mount_opts *mnt_opts;
        struct bpf_token *token = NULL;
        struct user_namespace *userns;
        struct inode *inode;
        struct file *file;
        CLASS(fd, f)(attr->token_create.bpffs_fd);
        struct path path;
        struct super_block *sb;
        umode_t mode;
        int err, fd;

        if (fd_empty(f))
                return -EBADF;

        path = fd_file(f)->f_path;
        sb = path.dentry->d_sb;

        if (path.dentry != sb->s_root)
                return -EINVAL;
        if (sb->s_op != &bpf_super_ops)
                return -EINVAL;
        err = path_permission(&path, MAY_ACCESS);
        if (err)
                return err;

        userns = sb->s_user_ns;
        /*
         * Enforce that creators of BPF tokens are in the same user
         * namespace as the BPF FS instance. This makes reasoning about
         * permissions a lot easier and we can always relax this later.
         */
        if (current_user_ns() != userns)
                return -EPERM;
        if (!ns_capable(userns, CAP_BPF))
                return -EPERM;

        /* Creating BPF token in init_user_ns doesn't make much sense. */
        if (current_user_ns() == &init_user_ns)
                return -EOPNOTSUPP;

        mnt_opts = sb->s_fs_info;
        if (mnt_opts->delegate_cmds == 0 &&
            mnt_opts->delegate_maps == 0 &&
            mnt_opts->delegate_progs == 0 &&
            mnt_opts->delegate_attachs == 0)
                return -ENOENT; /* no BPF token delegation is set up */

        mode = S_IFREG | ((S_IRUSR | S_IWUSR) & ~current_umask());
        inode = bpf_get_inode(sb, NULL, mode);
        if (IS_ERR(inode))
                return PTR_ERR(inode);

        inode->i_op = &bpf_token_iops;
        inode->i_fop = &bpf_token_fops;
        clear_nlink(inode); /* make sure it is unlinked */

        file = alloc_file_pseudo(inode, path.mnt, BPF_TOKEN_INODE_NAME, O_RDWR, &bpf_token_fops);
        if (IS_ERR(file)) {
                iput(inode);
                return PTR_ERR(file);
        }

        token = kzalloc(sizeof(*token), GFP_USER);
        if (!token) {
                err = -ENOMEM;
                goto out_file;
        }

        atomic64_set(&token->refcnt, 1);

        /* remember bpffs owning userns for future ns_capable() checks */
        token->userns = get_user_ns(userns);

        token->allowed_cmds = mnt_opts->delegate_cmds;
        token->allowed_maps = mnt_opts->delegate_maps;
        token->allowed_progs = mnt_opts->delegate_progs;
        token->allowed_attachs = mnt_opts->delegate_attachs;

        err = security_bpf_token_create(token, attr, &path);
        if (err)
                goto out_token;

        fd = get_unused_fd_flags(O_CLOEXEC);
        if (fd < 0) {
                err = fd;
                goto out_token;
        }

        file->private_data = token;
        fd_install(fd, file);

        return fd;

out_token:
        bpf_token_free(token);
out_file:
        fput(file);
        return err;
}

struct bpf_token *bpf_token_get_from_fd(u32 ufd)
{
        CLASS(fd, f)(ufd);
        struct bpf_token *token;

        if (fd_empty(f))
                return ERR_PTR(-EBADF);
        if (fd_file(f)->f_op != &bpf_token_fops)
                return ERR_PTR(-EINVAL);

        token = fd_file(f)->private_data;
        bpf_token_inc(token);

        return token;
}

bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd)
{
        if (!token)
                return false;
        if (!(token->allowed_cmds & BIT_ULL(cmd)))
                return false;
        return security_bpf_token_cmd(token, cmd) == 0;
}

bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type)
{
        if (!token || type >= __MAX_BPF_MAP_TYPE)
                return false;

        return token->allowed_maps & BIT_ULL(type);
}

bool bpf_token_allow_prog_type(const struct bpf_token *token,
                               enum bpf_prog_type prog_type,
                               enum bpf_attach_type attach_type)
{
        if (!token || prog_type >= __MAX_BPF_PROG_TYPE || attach_type >= __MAX_BPF_ATTACH_TYPE)
                return false;

        return (token->allowed_progs & BIT_ULL(prog_type)) &&
               (token->allowed_attachs & BIT_ULL(attach_type));
}

















































































































































































    5 
    7 


















































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Fast and scalable bitmaps.
 *
 * Copyright (C) 2016 Facebook
 * Copyright (C) 2013-2014 Jens Axboe
 */

#ifndef __LINUX_SCALE_BITMAP_H
#define __LINUX_SCALE_BITMAP_H

#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/cache.h>
#include <linux/list.h>
#include <linux/log2.h>
#include <linux/minmax.h>
#include <linux/percpu.h>
#include <linux/slab.h>
#include <linux/smp.h>
#include <linux/types.h>
#include <linux/wait.h>

struct seq_file;

/**
 * struct sbitmap_word - Word in a &struct sbitmap.
 */
struct sbitmap_word {
        /**
         * @word: word holding free bits
         */
        unsigned long word;

        /**
         * @cleared: word holding cleared bits
         */
        unsigned long cleared ____cacheline_aligned_in_smp;

        /**
         * @swap_lock: serializes simultaneous updates of ->word and ->cleared
         */
        raw_spinlock_t swap_lock;
} ____cacheline_aligned_in_smp;

/**
 * struct sbitmap - Scalable bitmap.
 *
 * A &struct sbitmap is spread over multiple cachelines to avoid ping-pong. This
 * trades off higher memory usage for better scalability.
 */
struct sbitmap {
        /**
         * @depth: Number of bits used in the whole bitmap.
         */
        unsigned int depth;

        /**
         * @shift: log2(number of bits used per word)
         */
        unsigned int shift;

        /**
         * @map_nr: Number of words (cachelines) being used for the bitmap.
         */
        unsigned int map_nr;

        /**
         * @round_robin: Allocate bits in strict round-robin order.
         */
        bool round_robin;

        /**
         * @map: Allocated bitmap.
         */
        struct sbitmap_word *map;

        /*
         * @alloc_hint: Cache of last successfully allocated or freed bit.
         *
         * This is per-cpu, which allows multiple users to stick to different
         * cachelines until the map is exhausted.
         */
        unsigned int __percpu *alloc_hint;
};

#define SBQ_WAIT_QUEUES 8
#define SBQ_WAKE_BATCH 8

/**
 * struct sbq_wait_state - Wait queue in a &struct sbitmap_queue.
 */
struct sbq_wait_state {
        /**
         * @wait: Wait queue.
         */
        wait_queue_head_t wait;
} ____cacheline_aligned_in_smp;

/**
 * struct sbitmap_queue - Scalable bitmap with the added ability to wait on free
 * bits.
 *
 * A &struct sbitmap_queue uses multiple wait queues and rolling wakeups to
 * avoid contention on the wait queue spinlock. This ensures that we don't hit a
 * scalability wall when we run out of free bits and have to start putting tasks
 * to sleep.
 */
struct sbitmap_queue {
        /**
         * @sb: Scalable bitmap.
         */
        struct sbitmap sb;

        /**
         * @wake_batch: Number of bits which must be freed before we wake up any
         * waiters.
         */
        unsigned int wake_batch;

        /**
         * @wake_index: Next wait queue in @ws to wake up.
         */
        atomic_t wake_index;

        /**
         * @ws: Wait queues.
         */
        struct sbq_wait_state *ws;

        /*
         * @ws_active: count of currently active ws waitqueues
         */
        atomic_t ws_active;

        /**
         * @min_shallow_depth: The minimum shallow depth which may be passed to
         * sbitmap_queue_get_shallow()
         */
        unsigned int min_shallow_depth;

        /**
         * @completion_cnt: Number of bits cleared passed to the
         * wakeup function.
         */
        atomic_t completion_cnt;

        /**
         * @wakeup_cnt: Number of thread wake ups issued.
         */
        atomic_t wakeup_cnt;
};

/**
 * sbitmap_init_node() - Initialize a &struct sbitmap on a specific memory node.
 * @sb: Bitmap to initialize.
 * @depth: Number of bits to allocate.
 * @shift: Use 2^@shift bits per word in the bitmap; if a negative number if
 *         given, a good default is chosen.
 * @flags: Allocation flags.
 * @node: Memory node to allocate on.
 * @round_robin: If true, be stricter about allocation order; always allocate
 *               starting from the last allocated bit. This is less efficient
 *               than the default behavior (false).
 * @alloc_hint: If true, apply percpu hint for where to start searching for
 *              a free bit.
 *
 * Return: Zero on success or negative errno on failure.
 */
int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift,
                      gfp_t flags, int node, bool round_robin, bool alloc_hint);

/* sbitmap internal helper */
static inline unsigned int __map_depth(const struct sbitmap *sb, int index)
{
        if (index == sb->map_nr - 1)
                return sb->depth - (index << sb->shift);
        return 1U << sb->shift;
}

/**
 * sbitmap_free() - Free memory used by a &struct sbitmap.
 * @sb: Bitmap to free.
 */
static inline void sbitmap_free(struct sbitmap *sb)
{
        free_percpu(sb->alloc_hint);
        kvfree(sb->map);
        sb->map = NULL;
}

/**
 * sbitmap_resize() - Resize a &struct sbitmap.
 * @sb: Bitmap to resize.
 * @depth: New number of bits to resize to.
 *
 * Doesn't reallocate anything. It's up to the caller to ensure that the new
 * depth doesn't exceed the depth that the sb was initialized with.
 */
void sbitmap_resize(struct sbitmap *sb, unsigned int depth);

/**
 * sbitmap_get() - Try to allocate a free bit from a &struct sbitmap.
 * @sb: Bitmap to allocate from.
 *
 * This operation provides acquire barrier semantics if it succeeds.
 *
 * Return: Non-negative allocated bit number if successful, -1 otherwise.
 */
int sbitmap_get(struct sbitmap *sb);

/**
 * sbitmap_get_shallow() - Try to allocate a free bit from a &struct sbitmap,
 * limiting the depth used from each word.
 * @sb: Bitmap to allocate from.
 * @shallow_depth: The maximum number of bits to allocate from a single word.
 *
 * This rather specific operation allows for having multiple users with
 * different allocation limits. E.g., there can be a high-priority class that
 * uses sbitmap_get() and a low-priority class that uses sbitmap_get_shallow()
 * with a @shallow_depth of (1 << (@sb->shift - 1)). Then, the low-priority
 * class can only allocate half of the total bits in the bitmap, preventing it
 * from starving out the high-priority class.
 *
 * Return: Non-negative allocated bit number if successful, -1 otherwise.
 */
int sbitmap_get_shallow(struct sbitmap *sb, unsigned long shallow_depth);

/**
 * sbitmap_any_bit_set() - Check for a set bit in a &struct sbitmap.
 * @sb: Bitmap to check.
 *
 * Return: true if any bit in the bitmap is set, false otherwise.
 */
bool sbitmap_any_bit_set(const struct sbitmap *sb);

#define SB_NR_TO_INDEX(sb, bitnr) ((bitnr) >> (sb)->shift)
#define SB_NR_TO_BIT(sb, bitnr) ((bitnr) & ((1U << (sb)->shift) - 1U))

typedef bool (*sb_for_each_fn)(struct sbitmap *, unsigned int, void *);

/**
 * __sbitmap_for_each_set() - Iterate over each set bit in a &struct sbitmap.
 * @start: Where to start the iteration.
 * @sb: Bitmap to iterate over.
 * @fn: Callback. Should return true to continue or false to break early.
 * @data: Pointer to pass to callback.
 *
 * This is inline even though it's non-trivial so that the function calls to the
 * callback will hopefully get optimized away.
 */
static inline void __sbitmap_for_each_set(struct sbitmap *sb,
                                          unsigned int start,
                                          sb_for_each_fn fn, void *data)
{
        unsigned int index;
        unsigned int nr;
        unsigned int scanned = 0;

        if (start >= sb->depth)
                start = 0;
        index = SB_NR_TO_INDEX(sb, start);
        nr = SB_NR_TO_BIT(sb, start);

        while (scanned < sb->depth) {
                unsigned long word;
                unsigned int depth = min_t(unsigned int,
                                           __map_depth(sb, index) - nr,
                                           sb->depth - scanned);

                scanned += depth;
                word = sb->map[index].word & ~sb->map[index].cleared;
                if (!word)
                        goto next;

                /*
                 * On the first iteration of the outer loop, we need to add the
                 * bit offset back to the size of the word for find_next_bit().
                 * On all other iterations, nr is zero, so this is a noop.
                 */
                depth += nr;
                while (1) {
                        nr = find_next_bit(&word, depth, nr);
                        if (nr >= depth)
                                break;
                        if (!fn(sb, (index << sb->shift) + nr, data))
                                return;

                        nr++;
                }
next:
                nr = 0;
                if (++index >= sb->map_nr)
                        index = 0;
        }
}

/**
 * sbitmap_for_each_set() - Iterate over each set bit in a &struct sbitmap.
 * @sb: Bitmap to iterate over.
 * @fn: Callback. Should return true to continue or false to break early.
 * @data: Pointer to pass to callback.
 */
static inline void sbitmap_for_each_set(struct sbitmap *sb, sb_for_each_fn fn,
                                        void *data)
{
        __sbitmap_for_each_set(sb, 0, fn, data);
}

static inline unsigned long *__sbitmap_word(struct sbitmap *sb,
                                            unsigned int bitnr)
{
        return &sb->map[SB_NR_TO_INDEX(sb, bitnr)].word;
}

/* Helpers equivalent to the operations in asm/bitops.h and linux/bitmap.h */

static inline void sbitmap_set_bit(struct sbitmap *sb, unsigned int bitnr)
{
        set_bit(SB_NR_TO_BIT(sb, bitnr), __sbitmap_word(sb, bitnr));
}

static inline void sbitmap_clear_bit(struct sbitmap *sb, unsigned int bitnr)
{
        clear_bit(SB_NR_TO_BIT(sb, bitnr), __sbitmap_word(sb, bitnr));
}

/*
 * This one is special, since it doesn't actually clear the bit, rather it
 * sets the corresponding bit in the ->cleared mask instead. Paired with
 * the caller doing sbitmap_deferred_clear() if a given index is full, which
 * will clear the previously freed entries in the corresponding ->word.
 */
static inline void sbitmap_deferred_clear_bit(struct sbitmap *sb, unsigned int bitnr)
{
        unsigned long *addr = &sb->map[SB_NR_TO_INDEX(sb, bitnr)].cleared;

        set_bit(SB_NR_TO_BIT(sb, bitnr), addr);
}

/*
 * Pair of sbitmap_get, and this one applies both cleared bit and
 * allocation hint.
 */
static inline void sbitmap_put(struct sbitmap *sb, unsigned int bitnr)
{
        sbitmap_deferred_clear_bit(sb, bitnr);

        if (likely(sb->alloc_hint && !sb->round_robin && bitnr < sb->depth))
                *raw_cpu_ptr(sb->alloc_hint) = bitnr;
}

static inline int sbitmap_test_bit(struct sbitmap *sb, unsigned int bitnr)
{
        return test_bit(SB_NR_TO_BIT(sb, bitnr), __sbitmap_word(sb, bitnr));
}

static inline int sbitmap_calculate_shift(unsigned int depth)
{
        int        shift = ilog2(BITS_PER_LONG);

        /*
         * If the bitmap is small, shrink the number of bits per word so
         * we spread over a few cachelines, at least. If less than 4
         * bits, just forget about it, it's not going to work optimally
         * anyway.
         */
        if (depth >= 4) {
                while ((4U << shift) > depth)
                        shift--;
        }

        return shift;
}

/**
 * sbitmap_show() - Dump &struct sbitmap information to a &struct seq_file.
 * @sb: Bitmap to show.
 * @m: struct seq_file to write to.
 *
 * This is intended for debugging. The format may change at any time.
 */
void sbitmap_show(struct sbitmap *sb, struct seq_file *m);


/**
 * sbitmap_weight() - Return how many set and not cleared bits in a &struct
 * sbitmap.
 * @sb: Bitmap to check.
 *
 * Return: How many set and not cleared bits set
 */
unsigned int sbitmap_weight(const struct sbitmap *sb);

/**
 * sbitmap_bitmap_show() - Write a hex dump of a &struct sbitmap to a &struct
 * seq_file.
 * @sb: Bitmap to show.
 * @m: struct seq_file to write to.
 *
 * This is intended for debugging. The output isn't guaranteed to be internally
 * consistent.
 */
void sbitmap_bitmap_show(struct sbitmap *sb, struct seq_file *m);

/**
 * sbitmap_queue_init_node() - Initialize a &struct sbitmap_queue on a specific
 * memory node.
 * @sbq: Bitmap queue to initialize.
 * @depth: See sbitmap_init_node().
 * @shift: See sbitmap_init_node().
 * @round_robin: See sbitmap_get().
 * @flags: Allocation flags.
 * @node: Memory node to allocate on.
 *
 * Return: Zero on success or negative errno on failure.
 */
int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth,
                            int shift, bool round_robin, gfp_t flags, int node);

/**
 * sbitmap_queue_free() - Free memory used by a &struct sbitmap_queue.
 *
 * @sbq: Bitmap queue to free.
 */
static inline void sbitmap_queue_free(struct sbitmap_queue *sbq)
{
        kfree(sbq->ws);
        sbitmap_free(&sbq->sb);
}

/**
 * sbitmap_queue_recalculate_wake_batch() - Recalculate wake batch
 * @sbq: Bitmap queue to recalculate wake batch.
 * @users: Number of shares.
 *
 * Like sbitmap_queue_update_wake_batch(), this will calculate wake batch
 * by depth. This interface is for HCTX shared tags or queue shared tags.
 */
void sbitmap_queue_recalculate_wake_batch(struct sbitmap_queue *sbq,
                                            unsigned int users);

/**
 * sbitmap_queue_resize() - Resize a &struct sbitmap_queue.
 * @sbq: Bitmap queue to resize.
 * @depth: New number of bits to resize to.
 *
 * Like sbitmap_resize(), this doesn't reallocate anything. It has to do
 * some extra work on the &struct sbitmap_queue, so it's not safe to just
 * resize the underlying &struct sbitmap.
 */
void sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth);

/**
 * __sbitmap_queue_get() - Try to allocate a free bit from a &struct
 * sbitmap_queue with preemption already disabled.
 * @sbq: Bitmap queue to allocate from.
 *
 * Return: Non-negative allocated bit number if successful, -1 otherwise.
 */
int __sbitmap_queue_get(struct sbitmap_queue *sbq);

/**
 * __sbitmap_queue_get_batch() - Try to allocate a batch of free bits
 * @sbq: Bitmap queue to allocate from.
 * @nr_tags: number of tags requested
 * @offset: offset to add to returned bits
 *
 * Return: Mask of allocated tags, 0 if none are found. Each tag allocated is
 * a bit in the mask returned, and the caller must add @offset to the value to
 * get the absolute tag value.
 */
unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags,
                                        unsigned int *offset);

/**
 * sbitmap_queue_get_shallow() - Try to allocate a free bit from a &struct
 * sbitmap_queue, limiting the depth used from each word, with preemption
 * already disabled.
 * @sbq: Bitmap queue to allocate from.
 * @shallow_depth: The maximum number of bits to allocate from a single word.
 * See sbitmap_get_shallow().
 *
 * If you call this, make sure to call sbitmap_queue_min_shallow_depth() after
 * initializing @sbq.
 *
 * Return: Non-negative allocated bit number if successful, -1 otherwise.
 */
int sbitmap_queue_get_shallow(struct sbitmap_queue *sbq,
                              unsigned int shallow_depth);

/**
 * sbitmap_queue_get() - Try to allocate a free bit from a &struct
 * sbitmap_queue.
 * @sbq: Bitmap queue to allocate from.
 * @cpu: Output parameter; will contain the CPU we ran on (e.g., to be passed to
 *       sbitmap_queue_clear()).
 *
 * Return: Non-negative allocated bit number if successful, -1 otherwise.
 */
static inline int sbitmap_queue_get(struct sbitmap_queue *sbq,
                                    unsigned int *cpu)
{
        int nr;

        *cpu = get_cpu();
        nr = __sbitmap_queue_get(sbq);
        put_cpu();
        return nr;
}

/**
 * sbitmap_queue_min_shallow_depth() - Inform a &struct sbitmap_queue of the
 * minimum shallow depth that will be used.
 * @sbq: Bitmap queue in question.
 * @min_shallow_depth: The minimum shallow depth that will be passed to
 * sbitmap_queue_get_shallow() or __sbitmap_queue_get_shallow().
 *
 * sbitmap_queue_clear() batches wakeups as an optimization. The batch size
 * depends on the depth of the bitmap. Since the shallow allocation functions
 * effectively operate with a different depth, the shallow depth must be taken
 * into account when calculating the batch size. This function must be called
 * with the minimum shallow depth that will be used. Failure to do so can result
 * in missed wakeups.
 */
void sbitmap_queue_min_shallow_depth(struct sbitmap_queue *sbq,
                                     unsigned int min_shallow_depth);

/**
 * sbitmap_queue_clear() - Free an allocated bit and wake up waiters on a
 * &struct sbitmap_queue.
 * @sbq: Bitmap to free from.
 * @nr: Bit number to free.
 * @cpu: CPU the bit was allocated on.
 */
void sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr,
                         unsigned int cpu);

/**
 * sbitmap_queue_clear_batch() - Free a batch of allocated bits
 * &struct sbitmap_queue.
 * @sbq: Bitmap to free from.
 * @offset: offset for each tag in array
 * @tags: array of tags
 * @nr_tags: number of tags in array
 */
void sbitmap_queue_clear_batch(struct sbitmap_queue *sbq, int offset,
                                int *tags, int nr_tags);

static inline int sbq_index_inc(int index)
{
        return (index + 1) & (SBQ_WAIT_QUEUES - 1);
}

static inline void sbq_index_atomic_inc(atomic_t *index)
{
        int old = atomic_read(index);
        int new = sbq_index_inc(old);
        atomic_cmpxchg(index, old, new);
}

/**
 * sbq_wait_ptr() - Get the next wait queue to use for a &struct
 * sbitmap_queue.
 * @sbq: Bitmap queue to wait on.
 * @wait_index: A counter per "user" of @sbq.
 */
static inline struct sbq_wait_state *sbq_wait_ptr(struct sbitmap_queue *sbq,
                                                  atomic_t *wait_index)
{
        struct sbq_wait_state *ws;

        ws = &sbq->ws[atomic_read(wait_index)];
        sbq_index_atomic_inc(wait_index);
        return ws;
}

/**
 * sbitmap_queue_wake_all() - Wake up everything waiting on a &struct
 * sbitmap_queue.
 * @sbq: Bitmap queue to wake up.
 */
void sbitmap_queue_wake_all(struct sbitmap_queue *sbq);

/**
 * sbitmap_queue_wake_up() - Wake up some of waiters in one waitqueue
 * on a &struct sbitmap_queue.
 * @sbq: Bitmap queue to wake up.
 * @nr: Number of bits cleared.
 */
void sbitmap_queue_wake_up(struct sbitmap_queue *sbq, int nr);

/**
 * sbitmap_queue_show() - Dump &struct sbitmap_queue information to a &struct
 * seq_file.
 * @sbq: Bitmap queue to show.
 * @m: struct seq_file to write to.
 *
 * This is intended for debugging. The format may change at any time.
 */
void sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m);

struct sbq_wait {
        struct sbitmap_queue *sbq;        /* if set, sbq_wait is accounted */
        struct wait_queue_entry wait;
};

#define DEFINE_SBQ_WAIT(name)                                                        \
        struct sbq_wait name = {                                                \
                .sbq = NULL,                                                        \
                .wait = {                                                        \
                        .private        = current,                                \
                        .func                = autoremove_wake_function,                \
                        .entry                = LIST_HEAD_INIT((name).wait.entry),        \
                }                                                                \
        }

/*
 * Wrapper around prepare_to_wait_exclusive(), which maintains some extra
 * internal state.
 */
void sbitmap_prepare_to_wait(struct sbitmap_queue *sbq,
                                struct sbq_wait_state *ws,
                                struct sbq_wait *sbq_wait, int state);

/*
 * Must be paired with sbitmap_prepare_to_wait().
 */
void sbitmap_finish_wait(struct sbitmap_queue *sbq, struct sbq_wait_state *ws,
                                struct sbq_wait *sbq_wait);

/*
 * Wrapper around add_wait_queue(), which maintains some extra internal state
 */
void sbitmap_add_wait_queue(struct sbitmap_queue *sbq,
                            struct sbq_wait_state *ws,
                            struct sbq_wait *sbq_wait);

/*
 * Must be paired with sbitmap_add_wait_queue()
 */
void sbitmap_del_wait_queue(struct sbq_wait *sbq_wait);

#endif /* __LINUX_SCALE_BITMAP_H */

































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * AppArmor security module
 *
 * This file contains AppArmor file mediation function definitions.
 *
 * Copyright (C) 1998-2008 Novell/SUSE
 * Copyright 2009-2010 Canonical Ltd.
 */

#ifndef __AA_FILE_H
#define __AA_FILE_H

#include <linux/spinlock.h>

#include "domain.h"
#include "match.h"
#include "perms.h"

struct aa_policydb;
struct aa_profile;
struct path;

#define mask_mode_t(X) (X & (MAY_EXEC | MAY_WRITE | MAY_READ | MAY_APPEND))

#define AA_AUDIT_FILE_MASK        (MAY_READ | MAY_WRITE | MAY_EXEC | MAY_APPEND |\
                                 AA_MAY_CREATE | AA_MAY_DELETE |        \
                                 AA_MAY_GETATTR | AA_MAY_SETATTR | \
                                 AA_MAY_CHMOD | AA_MAY_CHOWN | AA_MAY_LOCK | \
                                 AA_EXEC_MMAP | AA_MAY_LINK)

static inline struct aa_file_ctx *file_ctx(struct file *file)
{
        return file->f_security + apparmor_blob_sizes.lbs_file;
}

/* struct aa_file_ctx - the AppArmor context the file was opened in
 * @lock: lock to update the ctx
 * @label: label currently cached on the ctx
 * @perms: the permission the file was opened with
 */
struct aa_file_ctx {
        spinlock_t lock;
        struct aa_label __rcu *label;
        u32 allow;
};

/*
 * The xindex is broken into 3 parts
 * - index - an index into either the exec name table or the variable table
 * - exec type - which determines how the executable name and index are used
 * - flags - which modify how the destination name is applied
 */
#define AA_X_INDEX_MASK                AA_INDEX_MASK

#define AA_X_TYPE_MASK                0x0c000000
#define AA_X_NONE                AA_INDEX_NONE
#define AA_X_NAME                0x04000000 /* use executable name px */
#define AA_X_TABLE                0x08000000 /* use a specified name ->n# */

#define AA_X_UNSAFE                0x10000000
#define AA_X_CHILD                0x20000000
#define AA_X_INHERIT                0x40000000
#define AA_X_UNCONFINED                0x80000000

/* need to make conditional which ones are being set */
struct path_cond {
        kuid_t uid;
        umode_t mode;
};

#define COMBINED_PERM_MASK(X) ((X).allow | (X).audit | (X).quiet | (X).kill)

int aa_audit_file(const struct cred *cred,
                  struct aa_profile *profile, struct aa_perms *perms,
                  const char *op, u32 request, const char *name,
                  const char *target, struct aa_label *tlabel, kuid_t ouid,
                  const char *info, int error);

struct aa_perms *aa_lookup_fperms(struct aa_policydb *file_rules,
                                  aa_state_t state, struct path_cond *cond);
aa_state_t aa_str_perms(struct aa_policydb *file_rules, aa_state_t start,
                        const char *name, struct path_cond *cond,
                        struct aa_perms *perms);

int aa_path_perm(const char *op, const struct cred *subj_cred,
                 struct aa_label *label, const struct path *path,
                 int flags, u32 request, struct path_cond *cond);

int aa_path_link(const struct cred *subj_cred, struct aa_label *label,
                 struct dentry *old_dentry, const struct path *new_dir,
                 struct dentry *new_dentry);

int aa_file_perm(const char *op, const struct cred *subj_cred,
                 struct aa_label *label, struct file *file,
                 u32 request, bool in_atomic);

void aa_inherit_files(const struct cred *cred, struct files_struct *files);


/**
 * aa_map_file_perms - map file flags to AppArmor permissions
 * @file: open file to map flags to AppArmor permissions
 *
 * Returns: apparmor permission set for the file
 */
static inline u32 aa_map_file_to_perms(struct file *file)
{
        int flags = file->f_flags;
        u32 perms = 0;

        if (file->f_mode & FMODE_WRITE)
                perms |= MAY_WRITE;
        if (file->f_mode & FMODE_READ)
                perms |= MAY_READ;

        if ((flags & O_APPEND) && (perms & MAY_WRITE))
                perms = (perms & ~MAY_WRITE) | MAY_APPEND;
        /* trunc implies write permission */
        if (flags & O_TRUNC)
                perms |= MAY_WRITE;
        if (flags & O_CREAT)
                perms |= AA_MAY_CREATE;

        return perms;
}

#endif /* __AA_FILE_H */




















































  209 


    2 


  198 
   10 

  205 
  208 
  206 

  207 










   69 









   44 




   39 





































   70 









   23 
   46 



   34 
    8 

   34 


   42 





   69 


   21 











   49 



























   70 















  249 


   40 


  108 









   37 
  172 








   58 












   58 













   58 












   11 











   18 
   40 






   54 
    4 




    8 




    8 







    1 



   49 






   12 






   38 


   40 


   10 



    4 






   17 
   33 


   44 




   17 

   17 










   39 











   11 


































































   21 







































    5 













   25 

   21 


    5 

   12 






   13 


   13 

    5 




    6 














    2 


   11 
    8 



    3 










   10 









   10 



























   79 





   80 



   77 


    5 



   35 





    2 
   36 








































    3 



    5 































































































   17 







   27 




















   60 



   33 







   27 
   26 
   27 

    7 

   18 



   27 
   26 


   27 










   15 

   24 



























    2 




    3 





    2 


    3 




    1 

















































   17 



   44 


   10 



    7 








   19 





    8 




    5 












    7 

   27 
   24 



    2 




   29 































   59 














    3 
    4 


   53 






   53 




   53 


   53 







   53 



    4 



    9 




   37 























   15 


   42 











   41 




   42 
   42 


   42 





   42 























   42 















   41 











    2 


   57 







   55 









    1 


   55 





   38 

   33 


    4 


    1 




    9 


   25 




   25 






   55 



   25 




   25 


   55 





















  132 




    7 
  125 







  132 









   53 
   53 



  171 










  170 

    2 



  106 
   63 







  107 




    6 


   20 







   49 




  123 




   79 






















   79 










   79 





   18 














  145 



    1 




  126 







   18 



  110 



    1 


   60 

   49 


















































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * inet_diag.c        Module for monitoring INET transport protocols sockets.
 *
 * Authors:        Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/fcntl.h>
#include <linux/random.h>
#include <linux/slab.h>
#include <linux/cache.h>
#include <linux/init.h>
#include <linux/time.h>

#include <net/icmp.h>
#include <net/tcp.h>
#include <net/ipv6.h>
#include <net/inet_common.h>
#include <net/inet_connection_sock.h>
#include <net/inet_hashtables.h>
#include <net/inet_timewait_sock.h>
#include <net/inet6_hashtables.h>
#include <net/bpf_sk_storage.h>
#include <net/netlink.h>

#include <linux/inet.h>
#include <linux/stddef.h>

#include <linux/inet_diag.h>
#include <linux/sock_diag.h>

static const struct inet_diag_handler __rcu **inet_diag_table;

struct inet_diag_entry {
        const __be32 *saddr;
        const __be32 *daddr;
        u16 sport;
        u16 dport;
        u16 family;
        u16 userlocks;
        u32 ifindex;
        u32 mark;
#ifdef CONFIG_SOCK_CGROUP_DATA
        u64 cgroup_id;
#endif
};

static const struct inet_diag_handler *inet_diag_lock_handler(int proto)
{
        const struct inet_diag_handler *handler;

        if (proto < 0 || proto >= IPPROTO_MAX)
                return NULL;

        if (!READ_ONCE(inet_diag_table[proto]))
                sock_load_diag_module(AF_INET, proto);

        rcu_read_lock();
        handler = rcu_dereference(inet_diag_table[proto]);
        if (handler && !try_module_get(handler->owner))
                handler = NULL;
        rcu_read_unlock();

        return handler;
}

static void inet_diag_unlock_handler(const struct inet_diag_handler *handler)
{
        module_put(handler->owner);
}

void inet_diag_msg_common_fill(struct inet_diag_msg *r, struct sock *sk)
{
        r->idiag_family = sk->sk_family;

        r->id.idiag_sport = htons(sk->sk_num);
        r->id.idiag_dport = sk->sk_dport;
        r->id.idiag_if = sk->sk_bound_dev_if;
        sock_diag_save_cookie(sk, r->id.idiag_cookie);

#if IS_ENABLED(CONFIG_IPV6)
        if (sk->sk_family == AF_INET6) {
                *(struct in6_addr *)r->id.idiag_src = sk->sk_v6_rcv_saddr;
                *(struct in6_addr *)r->id.idiag_dst = sk->sk_v6_daddr;
        } else
#endif
        {
        memset(&r->id.idiag_src, 0, sizeof(r->id.idiag_src));
        memset(&r->id.idiag_dst, 0, sizeof(r->id.idiag_dst));

        r->id.idiag_src[0] = sk->sk_rcv_saddr;
        r->id.idiag_dst[0] = sk->sk_daddr;
        }
}
EXPORT_SYMBOL_GPL(inet_diag_msg_common_fill);

static size_t inet_sk_attr_size(struct sock *sk,
                                const struct inet_diag_req_v2 *req,
                                bool net_admin)
{
        const struct inet_diag_handler *handler;
        size_t aux = 0;

        rcu_read_lock();
        handler = rcu_dereference(inet_diag_table[req->sdiag_protocol]);
        DEBUG_NET_WARN_ON_ONCE(!handler);
        if (handler && handler->idiag_get_aux_size)
                aux = handler->idiag_get_aux_size(sk, net_admin);
        rcu_read_unlock();

        return          nla_total_size(sizeof(struct tcp_info))
                + nla_total_size(sizeof(struct inet_diag_msg))
                + inet_diag_msg_attrs_size()
                + nla_total_size(sizeof(struct inet_diag_meminfo))
                + nla_total_size(SK_MEMINFO_VARS * sizeof(u32))
                + nla_total_size(TCP_CA_NAME_MAX)
                + nla_total_size(sizeof(struct tcpvegas_info))
                + aux
                + 64;
}

int inet_diag_msg_attrs_fill(struct sock *sk, struct sk_buff *skb,
                             struct inet_diag_msg *r, int ext,
                             struct user_namespace *user_ns,
                             bool net_admin)
{
        const struct inet_sock *inet = inet_sk(sk);
        struct inet_diag_sockopt inet_sockopt;

        if (nla_put_u8(skb, INET_DIAG_SHUTDOWN, sk->sk_shutdown))
                goto errout;

        /* IPv6 dual-stack sockets use inet->tos for IPv4 connections,
         * hence this needs to be included regardless of socket family.
         */
        if (ext & (1 << (INET_DIAG_TOS - 1)))
                if (nla_put_u8(skb, INET_DIAG_TOS, READ_ONCE(inet->tos)) < 0)
                        goto errout;

#if IS_ENABLED(CONFIG_IPV6)
        if (r->idiag_family == AF_INET6) {
                if (ext & (1 << (INET_DIAG_TCLASS - 1)))
                        if (nla_put_u8(skb, INET_DIAG_TCLASS,
                                       inet6_sk(sk)->tclass) < 0)
                                goto errout;

                if (((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) &&
                    nla_put_u8(skb, INET_DIAG_SKV6ONLY, ipv6_only_sock(sk)))
                        goto errout;
        }
#endif

        if (net_admin && nla_put_u32(skb, INET_DIAG_MARK, READ_ONCE(sk->sk_mark)))
                goto errout;

        if (ext & (1 << (INET_DIAG_CLASS_ID - 1)) ||
            ext & (1 << (INET_DIAG_TCLASS - 1))) {
                u32 classid = 0;

#ifdef CONFIG_SOCK_CGROUP_DATA
                classid = sock_cgroup_classid(&sk->sk_cgrp_data);
#endif
                /* Fallback to socket priority if class id isn't set.
                 * Classful qdiscs use it as direct reference to class.
                 * For cgroup2 classid is always zero.
                 */
                if (!classid)
                        classid = READ_ONCE(sk->sk_priority);

                if (nla_put_u32(skb, INET_DIAG_CLASS_ID, classid))
                        goto errout;
        }

#ifdef CONFIG_SOCK_CGROUP_DATA
        if (nla_put_u64_64bit(skb, INET_DIAG_CGROUP_ID,
                              cgroup_id(sock_cgroup_ptr(&sk->sk_cgrp_data)),
                              INET_DIAG_PAD))
                goto errout;
#endif

        r->idiag_uid = from_kuid_munged(user_ns, sock_i_uid(sk));
        r->idiag_inode = sock_i_ino(sk);

        memset(&inet_sockopt, 0, sizeof(inet_sockopt));
        inet_sockopt.recverr        = inet_test_bit(RECVERR, sk);
        inet_sockopt.is_icsk        = inet_test_bit(IS_ICSK, sk);
        inet_sockopt.freebind        = inet_test_bit(FREEBIND, sk);
        inet_sockopt.hdrincl        = inet_test_bit(HDRINCL, sk);
        inet_sockopt.mc_loop        = inet_test_bit(MC_LOOP, sk);
        inet_sockopt.transparent = inet_test_bit(TRANSPARENT, sk);
        inet_sockopt.mc_all        = inet_test_bit(MC_ALL, sk);
        inet_sockopt.nodefrag        = inet_test_bit(NODEFRAG, sk);
        inet_sockopt.bind_address_no_port = inet_test_bit(BIND_ADDRESS_NO_PORT, sk);
        inet_sockopt.recverr_rfc4884 = inet_test_bit(RECVERR_RFC4884, sk);
        inet_sockopt.defer_connect = inet_test_bit(DEFER_CONNECT, sk);
        if (nla_put(skb, INET_DIAG_SOCKOPT, sizeof(inet_sockopt),
                    &inet_sockopt))
                goto errout;

        return 0;
errout:
        return 1;
}
EXPORT_SYMBOL_GPL(inet_diag_msg_attrs_fill);

static int inet_diag_parse_attrs(const struct nlmsghdr *nlh, int hdrlen,
                                 struct nlattr **req_nlas)
{
        struct nlattr *nla;
        int remaining;

        nlmsg_for_each_attr(nla, nlh, hdrlen, remaining) {
                int type = nla_type(nla);

                if (type == INET_DIAG_REQ_PROTOCOL && nla_len(nla) != sizeof(u32))
                        return -EINVAL;

                if (type < __INET_DIAG_REQ_MAX)
                        req_nlas[type] = nla;
        }
        return 0;
}

static int inet_diag_get_protocol(const struct inet_diag_req_v2 *req,
                                  const struct inet_diag_dump_data *data)
{
        if (data->req_nlas[INET_DIAG_REQ_PROTOCOL])
                return nla_get_u32(data->req_nlas[INET_DIAG_REQ_PROTOCOL]);
        return req->sdiag_protocol;
}

#define MAX_DUMP_ALLOC_SIZE (KMALLOC_MAX_SIZE - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))

int inet_sk_diag_fill(struct sock *sk, struct inet_connection_sock *icsk,
                      struct sk_buff *skb, struct netlink_callback *cb,
                      const struct inet_diag_req_v2 *req,
                      u16 nlmsg_flags, bool net_admin)
{
        const struct tcp_congestion_ops *ca_ops;
        const struct inet_diag_handler *handler;
        struct inet_diag_dump_data *cb_data;
        int ext = req->idiag_ext;
        struct inet_diag_msg *r;
        struct nlmsghdr  *nlh;
        struct nlattr *attr;
        void *info = NULL;
        u8 icsk_pending;
        int protocol;

        cb_data = cb->data;
        protocol = inet_diag_get_protocol(req, cb_data);

        /* inet_diag_lock_handler() made sure inet_diag_table[] is stable. */
        handler = rcu_dereference_protected(inet_diag_table[protocol], 1);
        DEBUG_NET_WARN_ON_ONCE(!handler);
        if (!handler)
                return -ENXIO;

        nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
                        cb->nlh->nlmsg_type, sizeof(*r), nlmsg_flags);
        if (!nlh)
                return -EMSGSIZE;

        r = nlmsg_data(nlh);
        BUG_ON(!sk_fullsock(sk));

        inet_diag_msg_common_fill(r, sk);
        r->idiag_state = sk->sk_state;
        r->idiag_timer = 0;
        r->idiag_retrans = 0;
        r->idiag_expires = 0;

        if (inet_diag_msg_attrs_fill(sk, skb, r, ext,
                                     sk_user_ns(NETLINK_CB(cb->skb).sk),
                                     net_admin))
                goto errout;

        if (ext & (1 << (INET_DIAG_MEMINFO - 1))) {
                struct inet_diag_meminfo minfo = {
                        .idiag_rmem = sk_rmem_alloc_get(sk),
                        .idiag_wmem = READ_ONCE(sk->sk_wmem_queued),
                        .idiag_fmem = READ_ONCE(sk->sk_forward_alloc),
                        .idiag_tmem = sk_wmem_alloc_get(sk),
                };

                if (nla_put(skb, INET_DIAG_MEMINFO, sizeof(minfo), &minfo) < 0)
                        goto errout;
        }

        if (ext & (1 << (INET_DIAG_SKMEMINFO - 1)))
                if (sock_diag_put_meminfo(sk, skb, INET_DIAG_SKMEMINFO))
                        goto errout;

        /*
         * RAW sockets might have user-defined protocols assigned,
         * so report the one supplied on socket creation.
         */
        if (sk->sk_type == SOCK_RAW) {
                if (nla_put_u8(skb, INET_DIAG_PROTOCOL, sk->sk_protocol))
                        goto errout;
        }

        if (!icsk) {
                handler->idiag_get_info(sk, r, NULL);
                goto out;
        }

        icsk_pending = smp_load_acquire(&icsk->icsk_pending);
        if (icsk_pending == ICSK_TIME_RETRANS ||
            icsk_pending == ICSK_TIME_REO_TIMEOUT ||
            icsk_pending == ICSK_TIME_LOSS_PROBE) {
                r->idiag_timer = 1;
                r->idiag_retrans = icsk->icsk_retransmits;
                r->idiag_expires =
                        jiffies_delta_to_msecs(icsk_timeout(icsk) - jiffies);
        } else if (icsk_pending == ICSK_TIME_PROBE0) {
                r->idiag_timer = 4;
                r->idiag_retrans = icsk->icsk_probes_out;
                r->idiag_expires =
                        jiffies_delta_to_msecs(icsk_timeout(icsk) - jiffies);
        } else if (timer_pending(&sk->sk_timer)) {
                r->idiag_timer = 2;
                r->idiag_retrans = icsk->icsk_probes_out;
                r->idiag_expires =
                        jiffies_delta_to_msecs(sk->sk_timer.expires - jiffies);
        }

        if ((ext & (1 << (INET_DIAG_INFO - 1))) && handler->idiag_info_size) {
                attr = nla_reserve_64bit(skb, INET_DIAG_INFO,
                                         handler->idiag_info_size,
                                         INET_DIAG_PAD);
                if (!attr)
                        goto errout;

                info = nla_data(attr);
        }

        if (ext & (1 << (INET_DIAG_CONG - 1))) {
                int err = 0;

                rcu_read_lock();
                ca_ops = READ_ONCE(icsk->icsk_ca_ops);
                if (ca_ops)
                        err = nla_put_string(skb, INET_DIAG_CONG, ca_ops->name);
                rcu_read_unlock();
                if (err < 0)
                        goto errout;
        }

        handler->idiag_get_info(sk, r, info);

        if (ext & (1 << (INET_DIAG_INFO - 1)) && handler->idiag_get_aux)
                if (handler->idiag_get_aux(sk, net_admin, skb) < 0)
                        goto errout;

        if (sk->sk_state < TCP_TIME_WAIT) {
                union tcp_cc_info info;
                size_t sz = 0;
                int attr;

                rcu_read_lock();
                ca_ops = READ_ONCE(icsk->icsk_ca_ops);
                if (ca_ops && ca_ops->get_info)
                        sz = ca_ops->get_info(sk, ext, &attr, &info);
                rcu_read_unlock();
                if (sz && nla_put(skb, attr, sz, &info) < 0)
                        goto errout;
        }

        /* Keep it at the end for potential retry with a larger skb,
         * or else do best-effort fitting, which is only done for the
         * first_nlmsg.
         */
        if (cb_data->bpf_stg_diag) {
                bool first_nlmsg = ((unsigned char *)nlh == skb->data);
                unsigned int prev_min_dump_alloc;
                unsigned int total_nla_size = 0;
                unsigned int msg_len;
                int err;

                msg_len = skb_tail_pointer(skb) - (unsigned char *)nlh;
                err = bpf_sk_storage_diag_put(cb_data->bpf_stg_diag, sk, skb,
                                              INET_DIAG_SK_BPF_STORAGES,
                                              &total_nla_size);

                if (!err)
                        goto out;

                total_nla_size += msg_len;
                prev_min_dump_alloc = cb->min_dump_alloc;
                if (total_nla_size > prev_min_dump_alloc)
                        cb->min_dump_alloc = min_t(u32, total_nla_size,
                                                   MAX_DUMP_ALLOC_SIZE);

                if (!first_nlmsg)
                        goto errout;

                if (cb->min_dump_alloc > prev_min_dump_alloc)
                        /* Retry with pskb_expand_head() with
                         * __GFP_DIRECT_RECLAIM
                         */
                        goto errout;

                WARN_ON_ONCE(total_nla_size <= prev_min_dump_alloc);

                /* Send what we have for this sk
                 * and move on to the next sk in the following
                 * dump()
                 */
        }

out:
        nlmsg_end(skb, nlh);
        return 0;

errout:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}
EXPORT_SYMBOL_GPL(inet_sk_diag_fill);

static int inet_twsk_diag_fill(struct sock *sk,
                               struct sk_buff *skb,
                               struct netlink_callback *cb,
                               u16 nlmsg_flags, bool net_admin)
{
        struct inet_timewait_sock *tw = inet_twsk(sk);
        struct inet_diag_msg *r;
        struct nlmsghdr *nlh;
        long tmo;

        nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid,
                        cb->nlh->nlmsg_seq, cb->nlh->nlmsg_type,
                        sizeof(*r), nlmsg_flags);
        if (!nlh)
                return -EMSGSIZE;

        r = nlmsg_data(nlh);
        BUG_ON(tw->tw_state != TCP_TIME_WAIT);

        inet_diag_msg_common_fill(r, sk);
        r->idiag_retrans      = 0;

        r->idiag_state              = READ_ONCE(tw->tw_substate);
        r->idiag_timer              = 3;
        tmo = tw->tw_timer.expires - jiffies;
        r->idiag_expires      = jiffies_delta_to_msecs(tmo);
        r->idiag_rqueue              = 0;
        r->idiag_wqueue              = 0;
        r->idiag_uid              = 0;
        r->idiag_inode              = 0;

        if (net_admin && nla_put_u32(skb, INET_DIAG_MARK,
                                     tw->tw_mark)) {
                nlmsg_cancel(skb, nlh);
                return -EMSGSIZE;
        }

        nlmsg_end(skb, nlh);
        return 0;
}

static int inet_req_diag_fill(struct sock *sk, struct sk_buff *skb,
                              struct netlink_callback *cb,
                              u16 nlmsg_flags, bool net_admin)
{
        struct request_sock *reqsk = inet_reqsk(sk);
        struct inet_diag_msg *r;
        struct nlmsghdr *nlh;
        long tmo;

        nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
                        cb->nlh->nlmsg_type, sizeof(*r), nlmsg_flags);
        if (!nlh)
                return -EMSGSIZE;

        r = nlmsg_data(nlh);
        inet_diag_msg_common_fill(r, sk);
        r->idiag_state = TCP_SYN_RECV;
        r->idiag_timer = 1;
        r->idiag_retrans = reqsk->num_retrans;

        BUILD_BUG_ON(offsetof(struct inet_request_sock, ir_cookie) !=
                     offsetof(struct sock, sk_cookie));

        tmo = inet_reqsk(sk)->rsk_timer.expires - jiffies;
        r->idiag_expires = jiffies_delta_to_msecs(tmo);
        r->idiag_rqueue        = 0;
        r->idiag_wqueue        = 0;
        r->idiag_uid        = 0;
        r->idiag_inode        = 0;

        if (net_admin && nla_put_u32(skb, INET_DIAG_MARK,
                                     inet_rsk(reqsk)->ir_mark)) {
                nlmsg_cancel(skb, nlh);
                return -EMSGSIZE;
        }

        nlmsg_end(skb, nlh);
        return 0;
}

static int sk_diag_fill(struct sock *sk, struct sk_buff *skb,
                        struct netlink_callback *cb,
                        const struct inet_diag_req_v2 *r,
                        u16 nlmsg_flags, bool net_admin)
{
        if (sk->sk_state == TCP_TIME_WAIT)
                return inet_twsk_diag_fill(sk, skb, cb, nlmsg_flags, net_admin);

        if (sk->sk_state == TCP_NEW_SYN_RECV)
                return inet_req_diag_fill(sk, skb, cb, nlmsg_flags, net_admin);

        return inet_sk_diag_fill(sk, inet_csk(sk), skb, cb, r, nlmsg_flags,
                                 net_admin);
}

struct sock *inet_diag_find_one_icsk(struct net *net,
                                     struct inet_hashinfo *hashinfo,
                                     const struct inet_diag_req_v2 *req)
{
        struct sock *sk;

        rcu_read_lock();
        if (req->sdiag_family == AF_INET)
                sk = inet_lookup(net, hashinfo, NULL, 0, req->id.idiag_dst[0],
                                 req->id.idiag_dport, req->id.idiag_src[0],
                                 req->id.idiag_sport, req->id.idiag_if);
#if IS_ENABLED(CONFIG_IPV6)
        else if (req->sdiag_family == AF_INET6) {
                if (ipv6_addr_v4mapped((struct in6_addr *)req->id.idiag_dst) &&
                    ipv6_addr_v4mapped((struct in6_addr *)req->id.idiag_src))
                        sk = inet_lookup(net, hashinfo, NULL, 0, req->id.idiag_dst[3],
                                         req->id.idiag_dport, req->id.idiag_src[3],
                                         req->id.idiag_sport, req->id.idiag_if);
                else
                        sk = inet6_lookup(net, hashinfo, NULL, 0,
                                          (struct in6_addr *)req->id.idiag_dst,
                                          req->id.idiag_dport,
                                          (struct in6_addr *)req->id.idiag_src,
                                          req->id.idiag_sport,
                                          req->id.idiag_if);
        }
#endif
        else {
                rcu_read_unlock();
                return ERR_PTR(-EINVAL);
        }
        rcu_read_unlock();
        if (!sk)
                return ERR_PTR(-ENOENT);

        if (sock_diag_check_cookie(sk, req->id.idiag_cookie)) {
                sock_gen_put(sk);
                return ERR_PTR(-ENOENT);
        }

        return sk;
}
EXPORT_SYMBOL_GPL(inet_diag_find_one_icsk);

int inet_diag_dump_one_icsk(struct inet_hashinfo *hashinfo,
                            struct netlink_callback *cb,
                            const struct inet_diag_req_v2 *req)
{
        struct sk_buff *in_skb = cb->skb;
        bool net_admin = netlink_net_capable(in_skb, CAP_NET_ADMIN);
        struct net *net = sock_net(in_skb->sk);
        struct sk_buff *rep;
        struct sock *sk;
        int err;

        sk = inet_diag_find_one_icsk(net, hashinfo, req);
        if (IS_ERR(sk))
                return PTR_ERR(sk);

        rep = nlmsg_new(inet_sk_attr_size(sk, req, net_admin), GFP_KERNEL);
        if (!rep) {
                err = -ENOMEM;
                goto out;
        }

        err = sk_diag_fill(sk, rep, cb, req, 0, net_admin);
        if (err < 0) {
                WARN_ON(err == -EMSGSIZE);
                nlmsg_free(rep);
                goto out;
        }
        err = nlmsg_unicast(net->diag_nlsk, rep, NETLINK_CB(in_skb).portid);

out:
        if (sk)
                sock_gen_put(sk);

        return err;
}
EXPORT_SYMBOL_GPL(inet_diag_dump_one_icsk);

static int inet_diag_cmd_exact(int cmd, struct sk_buff *in_skb,
                               const struct nlmsghdr *nlh,
                               int hdrlen,
                               const struct inet_diag_req_v2 *req)
{
        const struct inet_diag_handler *handler;
        struct inet_diag_dump_data dump_data;
        int err, protocol;

        memset(&dump_data, 0, sizeof(dump_data));
        err = inet_diag_parse_attrs(nlh, hdrlen, dump_data.req_nlas);
        if (err)
                return err;

        protocol = inet_diag_get_protocol(req, &dump_data);

        handler = inet_diag_lock_handler(protocol);
        if (!handler)
                return -ENOENT;

        if (cmd == SOCK_DIAG_BY_FAMILY) {
                struct netlink_callback cb = {
                        .nlh = nlh,
                        .skb = in_skb,
                        .data = &dump_data,
                };
                err = handler->dump_one(&cb, req);
        } else if (cmd == SOCK_DESTROY && handler->destroy) {
                err = handler->destroy(in_skb, req);
        } else {
                err = -EOPNOTSUPP;
        }
        inet_diag_unlock_handler(handler);

        return err;
}

static int bitstring_match(const __be32 *a1, const __be32 *a2, int bits)
{
        int words = bits >> 5;

        bits &= 0x1f;

        if (words) {
                if (memcmp(a1, a2, words << 2))
                        return 0;
        }
        if (bits) {
                __be32 w1, w2;
                __be32 mask;

                w1 = a1[words];
                w2 = a2[words];

                mask = htonl((0xffffffff) << (32 - bits));

                if ((w1 ^ w2) & mask)
                        return 0;
        }

        return 1;
}

static int inet_diag_bc_run(const struct nlattr *_bc,
                            const struct inet_diag_entry *entry)
{
        const void *bc = nla_data(_bc);
        int len = nla_len(_bc);

        while (len > 0) {
                int yes = 1;
                const struct inet_diag_bc_op *op = bc;

                switch (op->code) {
                case INET_DIAG_BC_NOP:
                        break;
                case INET_DIAG_BC_JMP:
                        yes = 0;
                        break;
                case INET_DIAG_BC_S_EQ:
                        yes = entry->sport == op[1].no;
                        break;
                case INET_DIAG_BC_S_GE:
                        yes = entry->sport >= op[1].no;
                        break;
                case INET_DIAG_BC_S_LE:
                        yes = entry->sport <= op[1].no;
                        break;
                case INET_DIAG_BC_D_EQ:
                        yes = entry->dport == op[1].no;
                        break;
                case INET_DIAG_BC_D_GE:
                        yes = entry->dport >= op[1].no;
                        break;
                case INET_DIAG_BC_D_LE:
                        yes = entry->dport <= op[1].no;
                        break;
                case INET_DIAG_BC_AUTO:
                        yes = !(entry->userlocks & SOCK_BINDPORT_LOCK);
                        break;
                case INET_DIAG_BC_S_COND:
                case INET_DIAG_BC_D_COND: {
                        const struct inet_diag_hostcond *cond;
                        const __be32 *addr;

                        cond = (const struct inet_diag_hostcond *)(op + 1);
                        if (cond->port != -1 &&
                            cond->port != (op->code == INET_DIAG_BC_S_COND ?
                                             entry->sport : entry->dport)) {
                                yes = 0;
                                break;
                        }

                        if (op->code == INET_DIAG_BC_S_COND)
                                addr = entry->saddr;
                        else
                                addr = entry->daddr;

                        if (cond->family != AF_UNSPEC &&
                            cond->family != entry->family) {
                                if (entry->family == AF_INET6 &&
                                    cond->family == AF_INET) {
                                        if (addr[0] == 0 && addr[1] == 0 &&
                                            addr[2] == htonl(0xffff) &&
                                            bitstring_match(addr + 3,
                                                            cond->addr,
                                                            cond->prefix_len))
                                                break;
                                }
                                yes = 0;
                                break;
                        }

                        if (cond->prefix_len == 0)
                                break;
                        if (bitstring_match(addr, cond->addr,
                                            cond->prefix_len))
                                break;
                        yes = 0;
                        break;
                }
                case INET_DIAG_BC_DEV_COND: {
                        u32 ifindex;

                        ifindex = *((const u32 *)(op + 1));
                        if (ifindex != entry->ifindex)
                                yes = 0;
                        break;
                }
                case INET_DIAG_BC_MARK_COND: {
                        struct inet_diag_markcond *cond;

                        cond = (struct inet_diag_markcond *)(op + 1);
                        if ((entry->mark & cond->mask) != cond->mark)
                                yes = 0;
                        break;
                }
#ifdef CONFIG_SOCK_CGROUP_DATA
                case INET_DIAG_BC_CGROUP_COND: {
                        u64 cgroup_id;

                        cgroup_id = get_unaligned((const u64 *)(op + 1));
                        if (cgroup_id != entry->cgroup_id)
                                yes = 0;
                        break;
                }
#endif
                }

                if (yes) {
                        len -= op->yes;
                        bc += op->yes;
                } else {
                        len -= op->no;
                        bc += op->no;
                }
        }
        return len == 0;
}

/* This helper is available for all sockets (ESTABLISH, TIMEWAIT, SYN_RECV)
 */
static void entry_fill_addrs(struct inet_diag_entry *entry,
                             const struct sock *sk)
{
#if IS_ENABLED(CONFIG_IPV6)
        if (sk->sk_family == AF_INET6) {
                entry->saddr = sk->sk_v6_rcv_saddr.s6_addr32;
                entry->daddr = sk->sk_v6_daddr.s6_addr32;
        } else
#endif
        {
                entry->saddr = &sk->sk_rcv_saddr;
                entry->daddr = &sk->sk_daddr;
        }
}

int inet_diag_bc_sk(const struct nlattr *bc, struct sock *sk)
{
        struct inet_sock *inet = inet_sk(sk);
        struct inet_diag_entry entry;

        if (!bc)
                return 1;

        entry.family = sk->sk_family;
        entry_fill_addrs(&entry, sk);
        entry.sport = inet->inet_num;
        entry.dport = ntohs(inet->inet_dport);
        entry.ifindex = sk->sk_bound_dev_if;
        entry.userlocks = sk_fullsock(sk) ? sk->sk_userlocks : 0;
        if (sk_fullsock(sk))
                entry.mark = READ_ONCE(sk->sk_mark);
        else if (sk->sk_state == TCP_NEW_SYN_RECV)
                entry.mark = inet_rsk(inet_reqsk(sk))->ir_mark;
        else if (sk->sk_state == TCP_TIME_WAIT)
                entry.mark = inet_twsk(sk)->tw_mark;
        else
                entry.mark = 0;
#ifdef CONFIG_SOCK_CGROUP_DATA
        entry.cgroup_id = sk_fullsock(sk) ?
                cgroup_id(sock_cgroup_ptr(&sk->sk_cgrp_data)) : 0;
#endif

        return inet_diag_bc_run(bc, &entry);
}
EXPORT_SYMBOL_GPL(inet_diag_bc_sk);

static int valid_cc(const void *bc, int len, int cc)
{
        while (len >= 0) {
                const struct inet_diag_bc_op *op = bc;

                if (cc > len)
                        return 0;
                if (cc == len)
                        return 1;
                if (op->yes < 4 || op->yes & 3)
                        return 0;
                len -= op->yes;
                bc  += op->yes;
        }
        return 0;
}

/* data is u32 ifindex */
static bool valid_devcond(const struct inet_diag_bc_op *op, int len,
                          int *min_len)
{
        /* Check ifindex space. */
        *min_len += sizeof(u32);
        if (len < *min_len)
                return false;

        return true;
}
/* Validate an inet_diag_hostcond. */
static bool valid_hostcond(const struct inet_diag_bc_op *op, int len,
                           int *min_len)
{
        struct inet_diag_hostcond *cond;
        int addr_len;

        /* Check hostcond space. */
        *min_len += sizeof(struct inet_diag_hostcond);
        if (len < *min_len)
                return false;
        cond = (struct inet_diag_hostcond *)(op + 1);

        /* Check address family and address length. */
        switch (cond->family) {
        case AF_UNSPEC:
                addr_len = 0;
                break;
        case AF_INET:
                addr_len = sizeof(struct in_addr);
                break;
        case AF_INET6:
                addr_len = sizeof(struct in6_addr);
                break;
        default:
                return false;
        }
        *min_len += addr_len;
        if (len < *min_len)
                return false;

        /* Check prefix length (in bits) vs address length (in bytes). */
        if (cond->prefix_len > 8 * addr_len)
                return false;

        return true;
}

/* Validate a port comparison operator. */
static bool valid_port_comparison(const struct inet_diag_bc_op *op,
                                  int len, int *min_len)
{
        /* Port comparisons put the port in a follow-on inet_diag_bc_op. */
        *min_len += sizeof(struct inet_diag_bc_op);
        if (len < *min_len)
                return false;
        return true;
}

static bool valid_markcond(const struct inet_diag_bc_op *op, int len,
                           int *min_len)
{
        *min_len += sizeof(struct inet_diag_markcond);
        return len >= *min_len;
}

#ifdef CONFIG_SOCK_CGROUP_DATA
static bool valid_cgroupcond(const struct inet_diag_bc_op *op, int len,
                             int *min_len)
{
        *min_len += sizeof(u64);
        return len >= *min_len;
}
#endif

static int inet_diag_bc_audit(const struct nlattr *attr,
                              const struct sk_buff *skb)
{
        bool net_admin = netlink_net_capable(skb, CAP_NET_ADMIN);
        const void *bytecode, *bc;
        int bytecode_len, len;

        if (!attr || nla_len(attr) < sizeof(struct inet_diag_bc_op))
                return -EINVAL;

        bytecode = bc = nla_data(attr);
        len = bytecode_len = nla_len(attr);

        while (len > 0) {
                int min_len = sizeof(struct inet_diag_bc_op);
                const struct inet_diag_bc_op *op = bc;

                switch (op->code) {
                case INET_DIAG_BC_S_COND:
                case INET_DIAG_BC_D_COND:
                        if (!valid_hostcond(bc, len, &min_len))
                                return -EINVAL;
                        break;
                case INET_DIAG_BC_DEV_COND:
                        if (!valid_devcond(bc, len, &min_len))
                                return -EINVAL;
                        break;
                case INET_DIAG_BC_S_EQ:
                case INET_DIAG_BC_S_GE:
                case INET_DIAG_BC_S_LE:
                case INET_DIAG_BC_D_EQ:
                case INET_DIAG_BC_D_GE:
                case INET_DIAG_BC_D_LE:
                        if (!valid_port_comparison(bc, len, &min_len))
                                return -EINVAL;
                        break;
                case INET_DIAG_BC_MARK_COND:
                        if (!net_admin)
                                return -EPERM;
                        if (!valid_markcond(bc, len, &min_len))
                                return -EINVAL;
                        break;
#ifdef CONFIG_SOCK_CGROUP_DATA
                case INET_DIAG_BC_CGROUP_COND:
                        if (!valid_cgroupcond(bc, len, &min_len))
                                return -EINVAL;
                        break;
#endif
                case INET_DIAG_BC_AUTO:
                case INET_DIAG_BC_JMP:
                case INET_DIAG_BC_NOP:
                        break;
                default:
                        return -EINVAL;
                }

                if (op->code != INET_DIAG_BC_NOP) {
                        if (op->no < min_len || op->no > len + 4 || op->no & 3)
                                return -EINVAL;
                        if (op->no < len &&
                            !valid_cc(bytecode, bytecode_len, len - op->no))
                                return -EINVAL;
                }

                if (op->yes < min_len || op->yes > len + 4 || op->yes & 3)
                        return -EINVAL;
                bc  += op->yes;
                len -= op->yes;
        }
        return len == 0 ? 0 : -EINVAL;
}

static void twsk_build_assert(void)
{
        BUILD_BUG_ON(offsetof(struct inet_timewait_sock, tw_family) !=
                     offsetof(struct sock, sk_family));

        BUILD_BUG_ON(offsetof(struct inet_timewait_sock, tw_num) !=
                     offsetof(struct inet_sock, inet_num));

        BUILD_BUG_ON(offsetof(struct inet_timewait_sock, tw_dport) !=
                     offsetof(struct inet_sock, inet_dport));

        BUILD_BUG_ON(offsetof(struct inet_timewait_sock, tw_rcv_saddr) !=
                     offsetof(struct inet_sock, inet_rcv_saddr));

        BUILD_BUG_ON(offsetof(struct inet_timewait_sock, tw_daddr) !=
                     offsetof(struct inet_sock, inet_daddr));

#if IS_ENABLED(CONFIG_IPV6)
        BUILD_BUG_ON(offsetof(struct inet_timewait_sock, tw_v6_rcv_saddr) !=
                     offsetof(struct sock, sk_v6_rcv_saddr));

        BUILD_BUG_ON(offsetof(struct inet_timewait_sock, tw_v6_daddr) !=
                     offsetof(struct sock, sk_v6_daddr));
#endif
}

void inet_diag_dump_icsk(struct inet_hashinfo *hashinfo, struct sk_buff *skb,
                         struct netlink_callback *cb,
                         const struct inet_diag_req_v2 *r)
{
        bool net_admin = netlink_net_capable(cb->skb, CAP_NET_ADMIN);
        struct inet_diag_dump_data *cb_data = cb->data;
        struct net *net = sock_net(skb->sk);
        u32 idiag_states = r->idiag_states;
        int i, num, s_i, s_num;
        struct nlattr *bc;
        struct sock *sk;

        bc = cb_data->inet_diag_nla_bc;
        if (idiag_states & TCPF_SYN_RECV)
                idiag_states |= TCPF_NEW_SYN_RECV;
        s_i = cb->args[1];
        s_num = num = cb->args[2];

        if (cb->args[0] == 0) {
                if (!(idiag_states & TCPF_LISTEN) || r->id.idiag_dport)
                        goto skip_listen_ht;

                for (i = s_i; i <= hashinfo->lhash2_mask; i++) {
                        struct inet_listen_hashbucket *ilb;
                        struct hlist_nulls_node *node;

                        num = 0;
                        ilb = &hashinfo->lhash2[i];

                        if (hlist_nulls_empty(&ilb->nulls_head)) {
                                s_num = 0;
                                continue;
                        }
                        spin_lock(&ilb->lock);
                        sk_nulls_for_each(sk, node, &ilb->nulls_head) {
                                struct inet_sock *inet = inet_sk(sk);

                                if (!net_eq(sock_net(sk), net))
                                        continue;

                                if (num < s_num) {
                                        num++;
                                        continue;
                                }

                                if (r->sdiag_family != AF_UNSPEC &&
                                    sk->sk_family != r->sdiag_family)
                                        goto next_listen;

                                if (r->id.idiag_sport != inet->inet_sport &&
                                    r->id.idiag_sport)
                                        goto next_listen;

                                if (!inet_diag_bc_sk(bc, sk))
                                        goto next_listen;

                                if (inet_sk_diag_fill(sk, inet_csk(sk), skb,
                                                      cb, r, NLM_F_MULTI,
                                                      net_admin) < 0) {
                                        spin_unlock(&ilb->lock);
                                        goto done;
                                }

next_listen:
                                ++num;
                        }
                        spin_unlock(&ilb->lock);

                        s_num = 0;
                }
skip_listen_ht:
                cb->args[0] = 1;
                s_i = num = s_num = 0;
        }

/* Process a maximum of SKARR_SZ sockets at a time when walking hash buckets
 * with bh disabled.
 */
#define SKARR_SZ 16

        /* Dump bound but inactive (not listening, connecting, etc.) sockets */
        if (cb->args[0] == 1) {
                if (!(idiag_states & TCPF_BOUND_INACTIVE))
                        goto skip_bind_ht;

                for (i = s_i; i < hashinfo->bhash_size; i++) {
                        struct inet_bind_hashbucket *ibb;
                        struct inet_bind2_bucket *tb2;
                        struct sock *sk_arr[SKARR_SZ];
                        int num_arr[SKARR_SZ];
                        int idx, accum, res;

resume_bind_walk:
                        num = 0;
                        accum = 0;
                        ibb = &hashinfo->bhash2[i];

                        if (hlist_empty(&ibb->chain)) {
                                s_num = 0;
                                continue;
                        }
                        spin_lock_bh(&ibb->lock);
                        inet_bind_bucket_for_each(tb2, &ibb->chain) {
                                if (!net_eq(ib2_net(tb2), net))
                                        continue;

                                sk_for_each_bound(sk, &tb2->owners) {
                                        struct inet_sock *inet = inet_sk(sk);

                                        if (num < s_num)
                                                goto next_bind;

                                        if (sk->sk_state != TCP_CLOSE ||
                                            !inet->inet_num)
                                                goto next_bind;

                                        if (r->sdiag_family != AF_UNSPEC &&
                                            r->sdiag_family != sk->sk_family)
                                                goto next_bind;

                                        if (!inet_diag_bc_sk(bc, sk))
                                                goto next_bind;

                                        sock_hold(sk);
                                        num_arr[accum] = num;
                                        sk_arr[accum] = sk;
                                        if (++accum == SKARR_SZ)
                                                goto pause_bind_walk;
next_bind:
                                        num++;
                                }
                        }
pause_bind_walk:
                        spin_unlock_bh(&ibb->lock);

                        res = 0;
                        for (idx = 0; idx < accum; idx++) {
                                if (res >= 0) {
                                        res = inet_sk_diag_fill(sk_arr[idx],
                                                                NULL, skb, cb,
                                                                r, NLM_F_MULTI,
                                                                net_admin);
                                        if (res < 0)
                                                num = num_arr[idx];
                                }
                                sock_put(sk_arr[idx]);
                        }
                        if (res < 0)
                                goto done;

                        cond_resched();

                        if (accum == SKARR_SZ) {
                                s_num = num + 1;
                                goto resume_bind_walk;
                        }

                        s_num = 0;
                }
skip_bind_ht:
                cb->args[0] = 2;
                s_i = num = s_num = 0;
        }

        if (!(idiag_states & ~TCPF_LISTEN))
                goto out;

        for (i = s_i; i <= hashinfo->ehash_mask; i++) {
                struct inet_ehash_bucket *head = &hashinfo->ehash[i];
                spinlock_t *lock = inet_ehash_lockp(hashinfo, i);
                struct hlist_nulls_node *node;
                struct sock *sk_arr[SKARR_SZ];
                int num_arr[SKARR_SZ];
                int idx, accum, res;

                if (hlist_nulls_empty(&head->chain))
                        continue;

                if (i > s_i)
                        s_num = 0;

next_chunk:
                num = 0;
                accum = 0;
                spin_lock_bh(lock);
                sk_nulls_for_each(sk, node, &head->chain) {
                        int state;

                        if (!net_eq(sock_net(sk), net))
                                continue;
                        if (num < s_num)
                                goto next_normal;
                        state = (sk->sk_state == TCP_TIME_WAIT) ?
                                READ_ONCE(inet_twsk(sk)->tw_substate) : sk->sk_state;
                        if (!(idiag_states & (1 << state)))
                                goto next_normal;
                        if (r->sdiag_family != AF_UNSPEC &&
                            sk->sk_family != r->sdiag_family)
                                goto next_normal;
                        if (r->id.idiag_sport != htons(sk->sk_num) &&
                            r->id.idiag_sport)
                                goto next_normal;
                        if (r->id.idiag_dport != sk->sk_dport &&
                            r->id.idiag_dport)
                                goto next_normal;
                        twsk_build_assert();

                        if (!inet_diag_bc_sk(bc, sk))
                                goto next_normal;

                        if (!refcount_inc_not_zero(&sk->sk_refcnt))
                                goto next_normal;

                        num_arr[accum] = num;
                        sk_arr[accum] = sk;
                        if (++accum == SKARR_SZ)
                                break;
next_normal:
                        ++num;
                }
                spin_unlock_bh(lock);
                res = 0;
                for (idx = 0; idx < accum; idx++) {
                        if (res >= 0) {
                                res = sk_diag_fill(sk_arr[idx], skb, cb, r,
                                                   NLM_F_MULTI, net_admin);
                                if (res < 0)
                                        num = num_arr[idx];
                        }
                        sock_gen_put(sk_arr[idx]);
                }
                if (res < 0)
                        break;
                cond_resched();
                if (accum == SKARR_SZ) {
                        s_num = num + 1;
                        goto next_chunk;
                }
        }

done:
        cb->args[1] = i;
        cb->args[2] = num;
out:
        ;
}
EXPORT_SYMBOL_GPL(inet_diag_dump_icsk);

static int __inet_diag_dump(struct sk_buff *skb, struct netlink_callback *cb,
                            const struct inet_diag_req_v2 *r)
{
        struct inet_diag_dump_data *cb_data = cb->data;
        const struct inet_diag_handler *handler;
        u32 prev_min_dump_alloc;
        int protocol, err = 0;

        protocol = inet_diag_get_protocol(r, cb_data);

again:
        prev_min_dump_alloc = cb->min_dump_alloc;
        handler = inet_diag_lock_handler(protocol);
        if (handler) {
                handler->dump(skb, cb, r);
                inet_diag_unlock_handler(handler);
        } else {
                err = -ENOENT;
        }
        /* The skb is not large enough to fit one sk info and
         * inet_sk_diag_fill() has requested for a larger skb.
         */
        if (!skb->len && cb->min_dump_alloc > prev_min_dump_alloc) {
                err = pskb_expand_head(skb, 0, cb->min_dump_alloc, GFP_KERNEL);
                if (!err)
                        goto again;
        }

        return err ? : skb->len;
}

static int inet_diag_dump(struct sk_buff *skb, struct netlink_callback *cb)
{
        return __inet_diag_dump(skb, cb, nlmsg_data(cb->nlh));
}

static int __inet_diag_dump_start(struct netlink_callback *cb, int hdrlen)
{
        const struct nlmsghdr *nlh = cb->nlh;
        struct inet_diag_dump_data *cb_data;
        struct sk_buff *skb = cb->skb;
        struct nlattr *nla;
        int err;

        cb_data = kzalloc(sizeof(*cb_data), GFP_KERNEL);
        if (!cb_data)
                return -ENOMEM;

        err = inet_diag_parse_attrs(nlh, hdrlen, cb_data->req_nlas);
        if (err) {
                kfree(cb_data);
                return err;
        }
        nla = cb_data->inet_diag_nla_bc;
        if (nla) {
                err = inet_diag_bc_audit(nla, skb);
                if (err) {
                        kfree(cb_data);
                        return err;
                }
        }

        nla = cb_data->inet_diag_nla_bpf_stgs;
        if (nla) {
                struct bpf_sk_storage_diag *bpf_stg_diag;

                bpf_stg_diag = bpf_sk_storage_diag_alloc(nla);
                if (IS_ERR(bpf_stg_diag)) {
                        kfree(cb_data);
                        return PTR_ERR(bpf_stg_diag);
                }
                cb_data->bpf_stg_diag = bpf_stg_diag;
        }

        cb->data = cb_data;
        return 0;
}

static int inet_diag_dump_start(struct netlink_callback *cb)
{
        return __inet_diag_dump_start(cb, sizeof(struct inet_diag_req_v2));
}

static int inet_diag_dump_start_compat(struct netlink_callback *cb)
{
        return __inet_diag_dump_start(cb, sizeof(struct inet_diag_req));
}

static int inet_diag_dump_done(struct netlink_callback *cb)
{
        struct inet_diag_dump_data *cb_data = cb->data;

        bpf_sk_storage_diag_free(cb_data->bpf_stg_diag);
        kfree(cb->data);

        return 0;
}

static int inet_diag_type2proto(int type)
{
        switch (type) {
        case TCPDIAG_GETSOCK:
                return IPPROTO_TCP;
        case DCCPDIAG_GETSOCK:
                return IPPROTO_DCCP;
        default:
                return 0;
        }
}

static int inet_diag_dump_compat(struct sk_buff *skb,
                                 struct netlink_callback *cb)
{
        struct inet_diag_req *rc = nlmsg_data(cb->nlh);
        struct inet_diag_req_v2 req;

        req.sdiag_family = AF_UNSPEC; /* compatibility */
        req.sdiag_protocol = inet_diag_type2proto(cb->nlh->nlmsg_type);
        req.idiag_ext = rc->idiag_ext;
        req.pad = 0;
        req.idiag_states = rc->idiag_states;
        req.id = rc->id;

        return __inet_diag_dump(skb, cb, &req);
}

static int inet_diag_get_exact_compat(struct sk_buff *in_skb,
                                      const struct nlmsghdr *nlh)
{
        struct inet_diag_req *rc = nlmsg_data(nlh);
        struct inet_diag_req_v2 req;

        req.sdiag_family = rc->idiag_family;
        req.sdiag_protocol = inet_diag_type2proto(nlh->nlmsg_type);
        req.idiag_ext = rc->idiag_ext;
        req.pad = 0;
        req.idiag_states = rc->idiag_states;
        req.id = rc->id;

        return inet_diag_cmd_exact(SOCK_DIAG_BY_FAMILY, in_skb, nlh,
                                   sizeof(struct inet_diag_req), &req);
}

static int inet_diag_rcv_msg_compat(struct sk_buff *skb, struct nlmsghdr *nlh)
{
        int hdrlen = sizeof(struct inet_diag_req);
        struct net *net = sock_net(skb->sk);

        if (nlh->nlmsg_type >= INET_DIAG_GETSOCK_MAX ||
            nlmsg_len(nlh) < hdrlen)
                return -EINVAL;

        if (nlh->nlmsg_flags & NLM_F_DUMP) {
                struct netlink_dump_control c = {
                        .start = inet_diag_dump_start_compat,
                        .done = inet_diag_dump_done,
                        .dump = inet_diag_dump_compat,
                };
                return netlink_dump_start(net->diag_nlsk, skb, nlh, &c);
        }

        return inet_diag_get_exact_compat(skb, nlh);
}

static int inet_diag_handler_cmd(struct sk_buff *skb, struct nlmsghdr *h)
{
        int hdrlen = sizeof(struct inet_diag_req_v2);
        struct net *net = sock_net(skb->sk);

        if (nlmsg_len(h) < hdrlen)
                return -EINVAL;

        if (h->nlmsg_type == SOCK_DIAG_BY_FAMILY &&
            h->nlmsg_flags & NLM_F_DUMP) {
                struct netlink_dump_control c = {
                        .start = inet_diag_dump_start,
                        .done = inet_diag_dump_done,
                        .dump = inet_diag_dump,
                };
                return netlink_dump_start(net->diag_nlsk, skb, h, &c);
        }

        return inet_diag_cmd_exact(h->nlmsg_type, skb, h, hdrlen,
                                   nlmsg_data(h));
}

static
int inet_diag_handler_get_info(struct sk_buff *skb, struct sock *sk)
{
        const struct inet_diag_handler *handler;
        struct nlmsghdr *nlh;
        struct nlattr *attr;
        struct inet_diag_msg *r;
        void *info = NULL;
        int err = 0;

        nlh = nlmsg_put(skb, 0, 0, SOCK_DIAG_BY_FAMILY, sizeof(*r), 0);
        if (!nlh)
                return -ENOMEM;

        r = nlmsg_data(nlh);
        memset(r, 0, sizeof(*r));
        inet_diag_msg_common_fill(r, sk);
        if (sk->sk_type == SOCK_DGRAM || sk->sk_type == SOCK_STREAM)
                r->id.idiag_sport = inet_sk(sk)->inet_sport;
        r->idiag_state = sk->sk_state;

        if ((err = nla_put_u8(skb, INET_DIAG_PROTOCOL, sk->sk_protocol))) {
                nlmsg_cancel(skb, nlh);
                return err;
        }

        handler = inet_diag_lock_handler(sk->sk_protocol);
        if (!handler) {
                nlmsg_cancel(skb, nlh);
                return -ENOENT;
        }

        attr = handler->idiag_info_size
                ? nla_reserve_64bit(skb, INET_DIAG_INFO,
                                    handler->idiag_info_size,
                                    INET_DIAG_PAD)
                : NULL;
        if (attr)
                info = nla_data(attr);

        handler->idiag_get_info(sk, r, info);
        inet_diag_unlock_handler(handler);

        nlmsg_end(skb, nlh);
        return 0;
}

static const struct sock_diag_handler inet_diag_handler = {
        .owner = THIS_MODULE,
        .family = AF_INET,
        .dump = inet_diag_handler_cmd,
        .get_info = inet_diag_handler_get_info,
        .destroy = inet_diag_handler_cmd,
};

static const struct sock_diag_handler inet6_diag_handler = {
        .owner = THIS_MODULE,
        .family = AF_INET6,
        .dump = inet_diag_handler_cmd,
        .get_info = inet_diag_handler_get_info,
        .destroy = inet_diag_handler_cmd,
};

int inet_diag_register(const struct inet_diag_handler *h)
{
        const __u16 type = h->idiag_type;

        if (type >= IPPROTO_MAX)
                return -EINVAL;

        return !cmpxchg((const struct inet_diag_handler **)&inet_diag_table[type],
                        NULL, h) ? 0 : -EEXIST;
}
EXPORT_SYMBOL_GPL(inet_diag_register);

void inet_diag_unregister(const struct inet_diag_handler *h)
{
        const __u16 type = h->idiag_type;

        if (type >= IPPROTO_MAX)
                return;

        xchg((const struct inet_diag_handler **)&inet_diag_table[type],
             NULL);
}
EXPORT_SYMBOL_GPL(inet_diag_unregister);

static const struct sock_diag_inet_compat inet_diag_compat = {
        .owner        = THIS_MODULE,
        .fn        = inet_diag_rcv_msg_compat,
};

static int __init inet_diag_init(void)
{
        const int inet_diag_table_size = (IPPROTO_MAX *
                                          sizeof(struct inet_diag_handler *));
        int err = -ENOMEM;

        inet_diag_table = kzalloc(inet_diag_table_size, GFP_KERNEL);
        if (!inet_diag_table)
                goto out;

        err = sock_diag_register(&inet_diag_handler);
        if (err)
                goto out_free_nl;

        err = sock_diag_register(&inet6_diag_handler);
        if (err)
                goto out_free_inet;

        sock_diag_register_inet_compat(&inet_diag_compat);
out:
        return err;

out_free_inet:
        sock_diag_unregister(&inet_diag_handler);
out_free_nl:
        kfree(inet_diag_table);
        goto out;
}

static void __exit inet_diag_exit(void)
{
        sock_diag_unregister(&inet6_diag_handler);
        sock_diag_unregister(&inet_diag_handler);
        sock_diag_unregister_inet_compat(&inet_diag_compat);
        kfree(inet_diag_table);
}

module_init(inet_diag_init);
module_exit(inet_diag_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("INET/INET6: socket monitoring via SOCK_DIAG");
MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, 2 /* AF_INET */);
MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, 10 /* AF_INET6 */);
























   10 









   10 
















   10 


   10 






   10 











































































   10 




   10 
















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
// SPDX-License-Identifier: GPL-2.0-or-later
/* Generic associative array implementation.
 *
 * See Documentation/core-api/assoc_array.rst for information.
 *
 * Copyright (C) 2013 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */
//#define DEBUG
#include <linux/rcupdate.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/assoc_array_priv.h>

/*
 * Iterate over an associative array.  The caller must hold the RCU read lock
 * or better.
 */
static int assoc_array_subtree_iterate(const struct assoc_array_ptr *root,
                                       const struct assoc_array_ptr *stop,
                                       int (*iterator)(const void *leaf,
                                                       void *iterator_data),
                                       void *iterator_data)
{
        const struct assoc_array_shortcut *shortcut;
        const struct assoc_array_node *node;
        const struct assoc_array_ptr *cursor, *ptr, *parent;
        unsigned long has_meta;
        int slot, ret;

        cursor = root;

begin_node:
        if (assoc_array_ptr_is_shortcut(cursor)) {
                /* Descend through a shortcut */
                shortcut = assoc_array_ptr_to_shortcut(cursor);
                cursor = READ_ONCE(shortcut->next_node); /* Address dependency. */
        }

        node = assoc_array_ptr_to_node(cursor);
        slot = 0;

        /* We perform two passes of each node.
         *
         * The first pass does all the leaves in this node.  This means we
         * don't miss any leaves if the node is split up by insertion whilst
         * we're iterating over the branches rooted here (we may, however, see
         * some leaves twice).
         */
        has_meta = 0;
        for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
                ptr = READ_ONCE(node->slots[slot]); /* Address dependency. */
                has_meta |= (unsigned long)ptr;
                if (ptr && assoc_array_ptr_is_leaf(ptr)) {
                        /* We need a barrier between the read of the pointer,
                         * which is supplied by the above READ_ONCE().
                         */
                        /* Invoke the callback */
                        ret = iterator(assoc_array_ptr_to_leaf(ptr),
                                       iterator_data);
                        if (ret)
                                return ret;
                }
        }

        /* The second pass attends to all the metadata pointers.  If we follow
         * one of these we may find that we don't come back here, but rather go
         * back to a replacement node with the leaves in a different layout.
         *
         * We are guaranteed to make progress, however, as the slot number for
         * a particular portion of the key space cannot change - and we
         * continue at the back pointer + 1.
         */
        if (!(has_meta & ASSOC_ARRAY_PTR_META_TYPE))
                goto finished_node;
        slot = 0;

continue_node:
        node = assoc_array_ptr_to_node(cursor);
        for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
                ptr = READ_ONCE(node->slots[slot]); /* Address dependency. */
                if (assoc_array_ptr_is_meta(ptr)) {
                        cursor = ptr;
                        goto begin_node;
                }
        }

finished_node:
        /* Move up to the parent (may need to skip back over a shortcut) */
        parent = READ_ONCE(node->back_pointer); /* Address dependency. */
        slot = node->parent_slot;
        if (parent == stop)
                return 0;

        if (assoc_array_ptr_is_shortcut(parent)) {
                shortcut = assoc_array_ptr_to_shortcut(parent);
                cursor = parent;
                parent = READ_ONCE(shortcut->back_pointer); /* Address dependency. */
                slot = shortcut->parent_slot;
                if (parent == stop)
                        return 0;
        }

        /* Ascend to next slot in parent node */
        cursor = parent;
        slot++;
        goto continue_node;
}

/**
 * assoc_array_iterate - Pass all objects in the array to a callback
 * @array: The array to iterate over.
 * @iterator: The callback function.
 * @iterator_data: Private data for the callback function.
 *
 * Iterate over all the objects in an associative array.  Each one will be
 * presented to the iterator function.
 *
 * If the array is being modified concurrently with the iteration then it is
 * possible that some objects in the array will be passed to the iterator
 * callback more than once - though every object should be passed at least
 * once.  If this is undesirable then the caller must lock against modification
 * for the duration of this function.
 *
 * The function will return 0 if no objects were in the array or else it will
 * return the result of the last iterator function called.  Iteration stops
 * immediately if any call to the iteration function results in a non-zero
 * return.
 *
 * The caller should hold the RCU read lock or better if concurrent
 * modification is possible.
 */
int assoc_array_iterate(const struct assoc_array *array,
                        int (*iterator)(const void *object,
                                        void *iterator_data),
                        void *iterator_data)
{
        struct assoc_array_ptr *root = READ_ONCE(array->root); /* Address dependency. */

        if (!root)
                return 0;
        return assoc_array_subtree_iterate(root, NULL, iterator, iterator_data);
}

enum assoc_array_walk_status {
        assoc_array_walk_tree_empty,
        assoc_array_walk_found_terminal_node,
        assoc_array_walk_found_wrong_shortcut,
};

struct assoc_array_walk_result {
        struct {
                struct assoc_array_node        *node;        /* Node in which leaf might be found */
                int                level;
                int                slot;
        } terminal_node;
        struct {
                struct assoc_array_shortcut *shortcut;
                int                level;
                int                sc_level;
                unsigned long        sc_segments;
                unsigned long        dissimilarity;
        } wrong_shortcut;
};

/*
 * Navigate through the internal tree looking for the closest node to the key.
 */
static enum assoc_array_walk_status
assoc_array_walk(const struct assoc_array *array,
                 const struct assoc_array_ops *ops,
                 const void *index_key,
                 struct assoc_array_walk_result *result)
{
        struct assoc_array_shortcut *shortcut;
        struct assoc_array_node *node;
        struct assoc_array_ptr *cursor, *ptr;
        unsigned long sc_segments, dissimilarity;
        unsigned long segments;
        int level, sc_level, next_sc_level;
        int slot;

        pr_devel("-->%s()\n", __func__);

        cursor = READ_ONCE(array->root);  /* Address dependency. */
        if (!cursor)
                return assoc_array_walk_tree_empty;

        level = 0;

        /* Use segments from the key for the new leaf to navigate through the
         * internal tree, skipping through nodes and shortcuts that are on
         * route to the destination.  Eventually we'll come to a slot that is
         * either empty or contains a leaf at which point we've found a node in
         * which the leaf we're looking for might be found or into which it
         * should be inserted.
         */
jumped:
        segments = ops->get_key_chunk(index_key, level);
        pr_devel("segments[%d]: %lx\n", level, segments);

        if (assoc_array_ptr_is_shortcut(cursor))
                goto follow_shortcut;

consider_node:
        node = assoc_array_ptr_to_node(cursor);
        slot = segments >> (level & ASSOC_ARRAY_KEY_CHUNK_MASK);
        slot &= ASSOC_ARRAY_FAN_MASK;
        ptr = READ_ONCE(node->slots[slot]); /* Address dependency. */

        pr_devel("consider slot %x [ix=%d type=%lu]\n",
                 slot, level, (unsigned long)ptr & 3);

        if (!assoc_array_ptr_is_meta(ptr)) {
                /* The node doesn't have a node/shortcut pointer in the slot
                 * corresponding to the index key that we have to follow.
                 */
                result->terminal_node.node = node;
                result->terminal_node.level = level;
                result->terminal_node.slot = slot;
                pr_devel("<--%s() = terminal_node\n", __func__);
                return assoc_array_walk_found_terminal_node;
        }

        if (assoc_array_ptr_is_node(ptr)) {
                /* There is a pointer to a node in the slot corresponding to
                 * this index key segment, so we need to follow it.
                 */
                cursor = ptr;
                level += ASSOC_ARRAY_LEVEL_STEP;
                if ((level & ASSOC_ARRAY_KEY_CHUNK_MASK) != 0)
                        goto consider_node;
                goto jumped;
        }

        /* There is a shortcut in the slot corresponding to the index key
         * segment.  We follow the shortcut if its partial index key matches
         * this leaf's.  Otherwise we need to split the shortcut.
         */
        cursor = ptr;
follow_shortcut:
        shortcut = assoc_array_ptr_to_shortcut(cursor);
        pr_devel("shortcut to %d\n", shortcut->skip_to_level);
        sc_level = level + ASSOC_ARRAY_LEVEL_STEP;
        BUG_ON(sc_level > shortcut->skip_to_level);

        do {
                /* Check the leaf against the shortcut's index key a word at a
                 * time, trimming the final word (the shortcut stores the index
                 * key completely from the root to the shortcut's target).
                 */
                if ((sc_level & ASSOC_ARRAY_KEY_CHUNK_MASK) == 0)
                        segments = ops->get_key_chunk(index_key, sc_level);

                sc_segments = shortcut->index_key[sc_level >> ASSOC_ARRAY_KEY_CHUNK_SHIFT];
                dissimilarity = segments ^ sc_segments;

                if (round_up(sc_level, ASSOC_ARRAY_KEY_CHUNK_SIZE) > shortcut->skip_to_level) {
                        /* Trim segments that are beyond the shortcut */
                        int shift = shortcut->skip_to_level & ASSOC_ARRAY_KEY_CHUNK_MASK;
                        dissimilarity &= ~(ULONG_MAX << shift);
                        next_sc_level = shortcut->skip_to_level;
                } else {
                        next_sc_level = sc_level + ASSOC_ARRAY_KEY_CHUNK_SIZE;
                        next_sc_level = round_down(next_sc_level, ASSOC_ARRAY_KEY_CHUNK_SIZE);
                }

                if (dissimilarity != 0) {
                        /* This shortcut points elsewhere */
                        result->wrong_shortcut.shortcut = shortcut;
                        result->wrong_shortcut.level = level;
                        result->wrong_shortcut.sc_level = sc_level;
                        result->wrong_shortcut.sc_segments = sc_segments;
                        result->wrong_shortcut.dissimilarity = dissimilarity;
                        return assoc_array_walk_found_wrong_shortcut;
                }

                sc_level = next_sc_level;
        } while (sc_level < shortcut->skip_to_level);

        /* The shortcut matches the leaf's index to this point. */
        cursor = READ_ONCE(shortcut->next_node); /* Address dependency. */
        if (((level ^ sc_level) & ~ASSOC_ARRAY_KEY_CHUNK_MASK) != 0) {
                level = sc_level;
                goto jumped;
        } else {
                level = sc_level;
                goto consider_node;
        }
}

/**
 * assoc_array_find - Find an object by index key
 * @array: The associative array to search.
 * @ops: The operations to use.
 * @index_key: The key to the object.
 *
 * Find an object in an associative array by walking through the internal tree
 * to the node that should contain the object and then searching the leaves
 * there.  NULL is returned if the requested object was not found in the array.
 *
 * The caller must hold the RCU read lock or better.
 */
void *assoc_array_find(const struct assoc_array *array,
                       const struct assoc_array_ops *ops,
                       const void *index_key)
{
        struct assoc_array_walk_result result;
        const struct assoc_array_node *node;
        const struct assoc_array_ptr *ptr;
        const void *leaf;
        int slot;

        if (assoc_array_walk(array, ops, index_key, &result) !=
            assoc_array_walk_found_terminal_node)
                return NULL;

        node = result.terminal_node.node;

        /* If the target key is available to us, it's has to be pointed to by
         * the terminal node.
         */
        for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
                ptr = READ_ONCE(node->slots[slot]); /* Address dependency. */
                if (ptr && assoc_array_ptr_is_leaf(ptr)) {
                        /* We need a barrier between the read of the pointer
                         * and dereferencing the pointer - but only if we are
                         * actually going to dereference it.
                         */
                        leaf = assoc_array_ptr_to_leaf(ptr);
                        if (ops->compare_object(leaf, index_key))
                                return (void *)leaf;
                }
        }

        return NULL;
}

/*
 * Destructively iterate over an associative array.  The caller must prevent
 * other simultaneous accesses.
 */
static void assoc_array_destroy_subtree(struct assoc_array_ptr *root,
                                        const struct assoc_array_ops *ops)
{
        struct assoc_array_shortcut *shortcut;
        struct assoc_array_node *node;
        struct assoc_array_ptr *cursor, *parent = NULL;
        int slot = -1;

        pr_devel("-->%s()\n", __func__);

        cursor = root;
        if (!cursor) {
                pr_devel("empty\n");
                return;
        }

move_to_meta:
        if (assoc_array_ptr_is_shortcut(cursor)) {
                /* Descend through a shortcut */
                pr_devel("[%d] shortcut\n", slot);
                BUG_ON(!assoc_array_ptr_is_shortcut(cursor));
                shortcut = assoc_array_ptr_to_shortcut(cursor);
                BUG_ON(shortcut->back_pointer != parent);
                BUG_ON(slot != -1 && shortcut->parent_slot != slot);
                parent = cursor;
                cursor = shortcut->next_node;
                slot = -1;
                BUG_ON(!assoc_array_ptr_is_node(cursor));
        }

        pr_devel("[%d] node\n", slot);
        node = assoc_array_ptr_to_node(cursor);
        BUG_ON(node->back_pointer != parent);
        BUG_ON(slot != -1 && node->parent_slot != slot);
        slot = 0;

continue_node:
        pr_devel("Node %p [back=%p]\n", node, node->back_pointer);
        for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
                struct assoc_array_ptr *ptr = node->slots[slot];
                if (!ptr)
                        continue;
                if (assoc_array_ptr_is_meta(ptr)) {
                        parent = cursor;
                        cursor = ptr;
                        goto move_to_meta;
                }

                if (ops) {
                        pr_devel("[%d] free leaf\n", slot);
                        ops->free_object(assoc_array_ptr_to_leaf(ptr));
                }
        }

        parent = node->back_pointer;
        slot = node->parent_slot;
        pr_devel("free node\n");
        kfree(node);
        if (!parent)
                return; /* Done */

        /* Move back up to the parent (may need to free a shortcut on
         * the way up) */
        if (assoc_array_ptr_is_shortcut(parent)) {
                shortcut = assoc_array_ptr_to_shortcut(parent);
                BUG_ON(shortcut->next_node != cursor);
                cursor = parent;
                parent = shortcut->back_pointer;
                slot = shortcut->parent_slot;
                pr_devel("free shortcut\n");
                kfree(shortcut);
                if (!parent)
                        return;

                BUG_ON(!assoc_array_ptr_is_node(parent));
        }

        /* Ascend to next slot in parent node */
        pr_devel("ascend to %p[%d]\n", parent, slot);
        cursor = parent;
        node = assoc_array_ptr_to_node(cursor);
        slot++;
        goto continue_node;
}

/**
 * assoc_array_destroy - Destroy an associative array
 * @array: The array to destroy.
 * @ops: The operations to use.
 *
 * Discard all metadata and free all objects in an associative array.  The
 * array will be empty and ready to use again upon completion.  This function
 * cannot fail.
 *
 * The caller must prevent all other accesses whilst this takes place as no
 * attempt is made to adjust pointers gracefully to permit RCU readlock-holding
 * accesses to continue.  On the other hand, no memory allocation is required.
 */
void assoc_array_destroy(struct assoc_array *array,
                         const struct assoc_array_ops *ops)
{
        assoc_array_destroy_subtree(array->root, ops);
        array->root = NULL;
}

/*
 * Handle insertion into an empty tree.
 */
static bool assoc_array_insert_in_empty_tree(struct assoc_array_edit *edit)
{
        struct assoc_array_node *new_n0;

        pr_devel("-->%s()\n", __func__);

        new_n0 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
        if (!new_n0)
                return false;

        edit->new_meta[0] = assoc_array_node_to_ptr(new_n0);
        edit->leaf_p = &new_n0->slots[0];
        edit->adjust_count_on = new_n0;
        edit->set[0].ptr = &edit->array->root;
        edit->set[0].to = assoc_array_node_to_ptr(new_n0);

        pr_devel("<--%s() = ok [no root]\n", __func__);
        return true;
}

/*
 * Handle insertion into a terminal node.
 */
static bool assoc_array_insert_into_terminal_node(struct assoc_array_edit *edit,
                                                  const struct assoc_array_ops *ops,
                                                  const void *index_key,
                                                  struct assoc_array_walk_result *result)
{
        struct assoc_array_shortcut *shortcut, *new_s0;
        struct assoc_array_node *node, *new_n0, *new_n1, *side;
        struct assoc_array_ptr *ptr;
        unsigned long dissimilarity, base_seg, blank;
        size_t keylen;
        bool have_meta;
        int level, diff;
        int slot, next_slot, free_slot, i, j;

        node        = result->terminal_node.node;
        level        = result->terminal_node.level;
        edit->segment_cache[ASSOC_ARRAY_FAN_OUT] = result->terminal_node.slot;

        pr_devel("-->%s()\n", __func__);

        /* We arrived at a node which doesn't have an onward node or shortcut
         * pointer that we have to follow.  This means that (a) the leaf we
         * want must go here (either by insertion or replacement) or (b) we
         * need to split this node and insert in one of the fragments.
         */
        free_slot = -1;

        /* Firstly, we have to check the leaves in this node to see if there's
         * a matching one we should replace in place.
         */
        for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
                ptr = node->slots[i];
                if (!ptr) {
                        free_slot = i;
                        continue;
                }
                if (assoc_array_ptr_is_leaf(ptr) &&
                    ops->compare_object(assoc_array_ptr_to_leaf(ptr),
                                        index_key)) {
                        pr_devel("replace in slot %d\n", i);
                        edit->leaf_p = &node->slots[i];
                        edit->dead_leaf = node->slots[i];
                        pr_devel("<--%s() = ok [replace]\n", __func__);
                        return true;
                }
        }

        /* If there is a free slot in this node then we can just insert the
         * leaf here.
         */
        if (free_slot >= 0) {
                pr_devel("insert in free slot %d\n", free_slot);
                edit->leaf_p = &node->slots[free_slot];
                edit->adjust_count_on = node;
                pr_devel("<--%s() = ok [insert]\n", __func__);
                return true;
        }

        /* The node has no spare slots - so we're either going to have to split
         * it or insert another node before it.
         *
         * Whatever, we're going to need at least two new nodes - so allocate
         * those now.  We may also need a new shortcut, but we deal with that
         * when we need it.
         */
        new_n0 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
        if (!new_n0)
                return false;
        edit->new_meta[0] = assoc_array_node_to_ptr(new_n0);
        new_n1 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
        if (!new_n1)
                return false;
        edit->new_meta[1] = assoc_array_node_to_ptr(new_n1);

        /* We need to find out how similar the leaves are. */
        pr_devel("no spare slots\n");
        have_meta = false;
        for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
                ptr = node->slots[i];
                if (assoc_array_ptr_is_meta(ptr)) {
                        edit->segment_cache[i] = 0xff;
                        have_meta = true;
                        continue;
                }
                base_seg = ops->get_object_key_chunk(
                        assoc_array_ptr_to_leaf(ptr), level);
                base_seg >>= level & ASSOC_ARRAY_KEY_CHUNK_MASK;
                edit->segment_cache[i] = base_seg & ASSOC_ARRAY_FAN_MASK;
        }

        if (have_meta) {
                pr_devel("have meta\n");
                goto split_node;
        }

        /* The node contains only leaves */
        dissimilarity = 0;
        base_seg = edit->segment_cache[0];
        for (i = 1; i < ASSOC_ARRAY_FAN_OUT; i++)
                dissimilarity |= edit->segment_cache[i] ^ base_seg;

        pr_devel("only leaves; dissimilarity=%lx\n", dissimilarity);

        if ((dissimilarity & ASSOC_ARRAY_FAN_MASK) == 0) {
                /* The old leaves all cluster in the same slot.  We will need
                 * to insert a shortcut if the new node wants to cluster with them.
                 */
                if ((edit->segment_cache[ASSOC_ARRAY_FAN_OUT] ^ base_seg) == 0)
                        goto all_leaves_cluster_together;

                /* Otherwise all the old leaves cluster in the same slot, but
                 * the new leaf wants to go into a different slot - so we
                 * create a new node (n0) to hold the new leaf and a pointer to
                 * a new node (n1) holding all the old leaves.
                 *
                 * This can be done by falling through to the node splitting
                 * path.
                 */
                pr_devel("present leaves cluster but not new leaf\n");
        }

split_node:
        pr_devel("split node\n");

        /* We need to split the current node.  The node must contain anything
         * from a single leaf (in the one leaf case, this leaf will cluster
         * with the new leaf) and the rest meta-pointers, to all leaves, some
         * of which may cluster.
         *
         * It won't contain the case in which all the current leaves plus the
         * new leaves want to cluster in the same slot.
         *
         * We need to expel at least two leaves out of a set consisting of the
         * leaves in the node and the new leaf.  The current meta pointers can
         * just be copied as they shouldn't cluster with any of the leaves.
         *
         * We need a new node (n0) to replace the current one and a new node to
         * take the expelled nodes (n1).
         */
        edit->set[0].to = assoc_array_node_to_ptr(new_n0);
        new_n0->back_pointer = node->back_pointer;
        new_n0->parent_slot = node->parent_slot;
        new_n1->back_pointer = assoc_array_node_to_ptr(new_n0);
        new_n1->parent_slot = -1; /* Need to calculate this */

do_split_node:
        pr_devel("do_split_node\n");

        new_n0->nr_leaves_on_branch = node->nr_leaves_on_branch;
        new_n1->nr_leaves_on_branch = 0;

        /* Begin by finding two matching leaves.  There have to be at least two
         * that match - even if there are meta pointers - because any leaf that
         * would match a slot with a meta pointer in it must be somewhere
         * behind that meta pointer and cannot be here.  Further, given N
         * remaining leaf slots, we now have N+1 leaves to go in them.
         */
        for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
                slot = edit->segment_cache[i];
                if (slot != 0xff)
                        for (j = i + 1; j < ASSOC_ARRAY_FAN_OUT + 1; j++)
                                if (edit->segment_cache[j] == slot)
                                        goto found_slot_for_multiple_occupancy;
        }
found_slot_for_multiple_occupancy:
        pr_devel("same slot: %x %x [%02x]\n", i, j, slot);
        BUG_ON(i >= ASSOC_ARRAY_FAN_OUT);
        BUG_ON(j >= ASSOC_ARRAY_FAN_OUT + 1);
        BUG_ON(slot >= ASSOC_ARRAY_FAN_OUT);

        new_n1->parent_slot = slot;

        /* Metadata pointers cannot change slot */
        for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++)
                if (assoc_array_ptr_is_meta(node->slots[i]))
                        new_n0->slots[i] = node->slots[i];
                else
                        new_n0->slots[i] = NULL;
        BUG_ON(new_n0->slots[slot] != NULL);
        new_n0->slots[slot] = assoc_array_node_to_ptr(new_n1);

        /* Filter the leaf pointers between the new nodes */
        free_slot = -1;
        next_slot = 0;
        for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
                if (assoc_array_ptr_is_meta(node->slots[i]))
                        continue;
                if (edit->segment_cache[i] == slot) {
                        new_n1->slots[next_slot++] = node->slots[i];
                        new_n1->nr_leaves_on_branch++;
                } else {
                        do {
                                free_slot++;
                        } while (new_n0->slots[free_slot] != NULL);
                        new_n0->slots[free_slot] = node->slots[i];
                }
        }

        pr_devel("filtered: f=%x n=%x\n", free_slot, next_slot);

        if (edit->segment_cache[ASSOC_ARRAY_FAN_OUT] != slot) {
                do {
                        free_slot++;
                } while (new_n0->slots[free_slot] != NULL);
                edit->leaf_p = &new_n0->slots[free_slot];
                edit->adjust_count_on = new_n0;
        } else {
                edit->leaf_p = &new_n1->slots[next_slot++];
                edit->adjust_count_on = new_n1;
        }

        BUG_ON(next_slot <= 1);

        edit->set_backpointers_to = assoc_array_node_to_ptr(new_n0);
        for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
                if (edit->segment_cache[i] == 0xff) {
                        ptr = node->slots[i];
                        BUG_ON(assoc_array_ptr_is_leaf(ptr));
                        if (assoc_array_ptr_is_node(ptr)) {
                                side = assoc_array_ptr_to_node(ptr);
                                edit->set_backpointers[i] = &side->back_pointer;
                        } else {
                                shortcut = assoc_array_ptr_to_shortcut(ptr);
                                edit->set_backpointers[i] = &shortcut->back_pointer;
                        }
                }
        }

        ptr = node->back_pointer;
        if (!ptr)
                edit->set[0].ptr = &edit->array->root;
        else if (assoc_array_ptr_is_node(ptr))
                edit->set[0].ptr = &assoc_array_ptr_to_node(ptr)->slots[node->parent_slot];
        else
                edit->set[0].ptr = &assoc_array_ptr_to_shortcut(ptr)->next_node;
        edit->excised_meta[0] = assoc_array_node_to_ptr(node);
        pr_devel("<--%s() = ok [split node]\n", __func__);
        return true;

all_leaves_cluster_together:
        /* All the leaves, new and old, want to cluster together in this node
         * in the same slot, so we have to replace this node with a shortcut to
         * skip over the identical parts of the key and then place a pair of
         * nodes, one inside the other, at the end of the shortcut and
         * distribute the keys between them.
         *
         * Firstly we need to work out where the leaves start diverging as a
         * bit position into their keys so that we know how big the shortcut
         * needs to be.
         *
         * We only need to make a single pass of N of the N+1 leaves because if
         * any keys differ between themselves at bit X then at least one of
         * them must also differ with the base key at bit X or before.
         */
        pr_devel("all leaves cluster together\n");
        diff = INT_MAX;
        for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
                int x = ops->diff_objects(assoc_array_ptr_to_leaf(node->slots[i]),
                                          index_key);
                if (x < diff) {
                        BUG_ON(x < 0);
                        diff = x;
                }
        }
        BUG_ON(diff == INT_MAX);
        BUG_ON(diff < level + ASSOC_ARRAY_LEVEL_STEP);

        keylen = round_up(diff, ASSOC_ARRAY_KEY_CHUNK_SIZE);
        keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT;

        new_s0 = kzalloc(struct_size(new_s0, index_key, keylen), GFP_KERNEL);
        if (!new_s0)
                return false;
        edit->new_meta[2] = assoc_array_shortcut_to_ptr(new_s0);

        edit->set[0].to = assoc_array_shortcut_to_ptr(new_s0);
        new_s0->back_pointer = node->back_pointer;
        new_s0->parent_slot = node->parent_slot;
        new_s0->next_node = assoc_array_node_to_ptr(new_n0);
        new_n0->back_pointer = assoc_array_shortcut_to_ptr(new_s0);
        new_n0->parent_slot = 0;
        new_n1->back_pointer = assoc_array_node_to_ptr(new_n0);
        new_n1->parent_slot = -1; /* Need to calculate this */

        new_s0->skip_to_level = level = diff & ~ASSOC_ARRAY_LEVEL_STEP_MASK;
        pr_devel("skip_to_level = %d [diff %d]\n", level, diff);
        BUG_ON(level <= 0);

        for (i = 0; i < keylen; i++)
                new_s0->index_key[i] =
                        ops->get_key_chunk(index_key, i * ASSOC_ARRAY_KEY_CHUNK_SIZE);

        if (level & ASSOC_ARRAY_KEY_CHUNK_MASK) {
                blank = ULONG_MAX << (level & ASSOC_ARRAY_KEY_CHUNK_MASK);
                pr_devel("blank off [%zu] %d: %lx\n", keylen - 1, level, blank);
                new_s0->index_key[keylen - 1] &= ~blank;
        }

        /* This now reduces to a node splitting exercise for which we'll need
         * to regenerate the disparity table.
         */
        for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
                ptr = node->slots[i];
                base_seg = ops->get_object_key_chunk(assoc_array_ptr_to_leaf(ptr),
                                                     level);
                base_seg >>= level & ASSOC_ARRAY_KEY_CHUNK_MASK;
                edit->segment_cache[i] = base_seg & ASSOC_ARRAY_FAN_MASK;
        }

        base_seg = ops->get_key_chunk(index_key, level);
        base_seg >>= level & ASSOC_ARRAY_KEY_CHUNK_MASK;
        edit->segment_cache[ASSOC_ARRAY_FAN_OUT] = base_seg & ASSOC_ARRAY_FAN_MASK;
        goto do_split_node;
}

/*
 * Handle insertion into the middle of a shortcut.
 */
static bool assoc_array_insert_mid_shortcut(struct assoc_array_edit *edit,
                                            const struct assoc_array_ops *ops,
                                            struct assoc_array_walk_result *result)
{
        struct assoc_array_shortcut *shortcut, *new_s0, *new_s1;
        struct assoc_array_node *node, *new_n0, *side;
        unsigned long sc_segments, dissimilarity, blank;
        size_t keylen;
        int level, sc_level, diff;
        int sc_slot;

        shortcut        = result->wrong_shortcut.shortcut;
        level                = result->wrong_shortcut.level;
        sc_level        = result->wrong_shortcut.sc_level;
        sc_segments        = result->wrong_shortcut.sc_segments;
        dissimilarity        = result->wrong_shortcut.dissimilarity;

        pr_devel("-->%s(ix=%d dis=%lx scix=%d)\n",
                 __func__, level, dissimilarity, sc_level);

        /* We need to split a shortcut and insert a node between the two
         * pieces.  Zero-length pieces will be dispensed with entirely.
         *
         * First of all, we need to find out in which level the first
         * difference was.
         */
        diff = __ffs(dissimilarity);
        diff &= ~ASSOC_ARRAY_LEVEL_STEP_MASK;
        diff += sc_level & ~ASSOC_ARRAY_KEY_CHUNK_MASK;
        pr_devel("diff=%d\n", diff);

        if (!shortcut->back_pointer) {
                edit->set[0].ptr = &edit->array->root;
        } else if (assoc_array_ptr_is_node(shortcut->back_pointer)) {
                node = assoc_array_ptr_to_node(shortcut->back_pointer);
                edit->set[0].ptr = &node->slots[shortcut->parent_slot];
        } else {
                BUG();
        }

        edit->excised_meta[0] = assoc_array_shortcut_to_ptr(shortcut);

        /* Create a new node now since we're going to need it anyway */
        new_n0 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
        if (!new_n0)
                return false;
        edit->new_meta[0] = assoc_array_node_to_ptr(new_n0);
        edit->adjust_count_on = new_n0;

        /* Insert a new shortcut before the new node if this segment isn't of
         * zero length - otherwise we just connect the new node directly to the
         * parent.
         */
        level += ASSOC_ARRAY_LEVEL_STEP;
        if (diff > level) {
                pr_devel("pre-shortcut %d...%d\n", level, diff);
                keylen = round_up(diff, ASSOC_ARRAY_KEY_CHUNK_SIZE);
                keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT;

                new_s0 = kzalloc(struct_size(new_s0, index_key, keylen),
                                 GFP_KERNEL);
                if (!new_s0)
                        return false;
                edit->new_meta[1] = assoc_array_shortcut_to_ptr(new_s0);
                edit->set[0].to = assoc_array_shortcut_to_ptr(new_s0);
                new_s0->back_pointer = shortcut->back_pointer;
                new_s0->parent_slot = shortcut->parent_slot;
                new_s0->next_node = assoc_array_node_to_ptr(new_n0);
                new_s0->skip_to_level = diff;

                new_n0->back_pointer = assoc_array_shortcut_to_ptr(new_s0);
                new_n0->parent_slot = 0;

                memcpy(new_s0->index_key, shortcut->index_key,
                       flex_array_size(new_s0, index_key, keylen));

                blank = ULONG_MAX << (diff & ASSOC_ARRAY_KEY_CHUNK_MASK);
                pr_devel("blank off [%zu] %d: %lx\n", keylen - 1, diff, blank);
                new_s0->index_key[keylen - 1] &= ~blank;
        } else {
                pr_devel("no pre-shortcut\n");
                edit->set[0].to = assoc_array_node_to_ptr(new_n0);
                new_n0->back_pointer = shortcut->back_pointer;
                new_n0->parent_slot = shortcut->parent_slot;
        }

        side = assoc_array_ptr_to_node(shortcut->next_node);
        new_n0->nr_leaves_on_branch = side->nr_leaves_on_branch;

        /* We need to know which slot in the new node is going to take a
         * metadata pointer.
         */
        sc_slot = sc_segments >> (diff & ASSOC_ARRAY_KEY_CHUNK_MASK);
        sc_slot &= ASSOC_ARRAY_FAN_MASK;

        pr_devel("new slot %lx >> %d -> %d\n",
                 sc_segments, diff & ASSOC_ARRAY_KEY_CHUNK_MASK, sc_slot);

        /* Determine whether we need to follow the new node with a replacement
         * for the current shortcut.  We could in theory reuse the current
         * shortcut if its parent slot number doesn't change - but that's a
         * 1-in-16 chance so not worth expending the code upon.
         */
        level = diff + ASSOC_ARRAY_LEVEL_STEP;
        if (level < shortcut->skip_to_level) {
                pr_devel("post-shortcut %d...%d\n", level, shortcut->skip_to_level);
                keylen = round_up(shortcut->skip_to_level, ASSOC_ARRAY_KEY_CHUNK_SIZE);
                keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT;

                new_s1 = kzalloc(struct_size(new_s1, index_key, keylen),
                                 GFP_KERNEL);
                if (!new_s1)
                        return false;
                edit->new_meta[2] = assoc_array_shortcut_to_ptr(new_s1);

                new_s1->back_pointer = assoc_array_node_to_ptr(new_n0);
                new_s1->parent_slot = sc_slot;
                new_s1->next_node = shortcut->next_node;
                new_s1->skip_to_level = shortcut->skip_to_level;

                new_n0->slots[sc_slot] = assoc_array_shortcut_to_ptr(new_s1);

                memcpy(new_s1->index_key, shortcut->index_key,
                       flex_array_size(new_s1, index_key, keylen));

                edit->set[1].ptr = &side->back_pointer;
                edit->set[1].to = assoc_array_shortcut_to_ptr(new_s1);
        } else {
                pr_devel("no post-shortcut\n");

                /* We don't have to replace the pointed-to node as long as we
                 * use memory barriers to make sure the parent slot number is
                 * changed before the back pointer (the parent slot number is
                 * irrelevant to the old parent shortcut).
                 */
                new_n0->slots[sc_slot] = shortcut->next_node;
                edit->set_parent_slot[0].p = &side->parent_slot;
                edit->set_parent_slot[0].to = sc_slot;
                edit->set[1].ptr = &side->back_pointer;
                edit->set[1].to = assoc_array_node_to_ptr(new_n0);
        }

        /* Install the new leaf in a spare slot in the new node. */
        if (sc_slot == 0)
                edit->leaf_p = &new_n0->slots[1];
        else
                edit->leaf_p = &new_n0->slots[0];

        pr_devel("<--%s() = ok [split shortcut]\n", __func__);
        return true;
}

/**
 * assoc_array_insert - Script insertion of an object into an associative array
 * @array: The array to insert into.
 * @ops: The operations to use.
 * @index_key: The key to insert at.
 * @object: The object to insert.
 *
 * Precalculate and preallocate a script for the insertion or replacement of an
 * object in an associative array.  This results in an edit script that can
 * either be applied or cancelled.
 *
 * The function returns a pointer to an edit script or -ENOMEM.
 *
 * The caller should lock against other modifications and must continue to hold
 * the lock until assoc_array_apply_edit() has been called.
 *
 * Accesses to the tree may take place concurrently with this function,
 * provided they hold the RCU read lock.
 */
struct assoc_array_edit *assoc_array_insert(struct assoc_array *array,
                                            const struct assoc_array_ops *ops,
                                            const void *index_key,
                                            void *object)
{
        struct assoc_array_walk_result result;
        struct assoc_array_edit *edit;

        pr_devel("-->%s()\n", __func__);

        /* The leaf pointer we're given must not have the bottom bit set as we
         * use those for type-marking the pointer.  NULL pointers are also not
         * allowed as they indicate an empty slot but we have to allow them
         * here as they can be updated later.
         */
        BUG_ON(assoc_array_ptr_is_meta(object));

        edit = kzalloc(sizeof(struct assoc_array_edit), GFP_KERNEL);
        if (!edit)
                return ERR_PTR(-ENOMEM);
        edit->array = array;
        edit->ops = ops;
        edit->leaf = assoc_array_leaf_to_ptr(object);
        edit->adjust_count_by = 1;

        switch (assoc_array_walk(array, ops, index_key, &result)) {
        case assoc_array_walk_tree_empty:
                /* Allocate a root node if there isn't one yet */
                if (!assoc_array_insert_in_empty_tree(edit))
                        goto enomem;
                return edit;

        case assoc_array_walk_found_terminal_node:
                /* We found a node that doesn't have a node/shortcut pointer in
                 * the slot corresponding to the index key that we have to
                 * follow.
                 */
                if (!assoc_array_insert_into_terminal_node(edit, ops, index_key,
                                                           &result))
                        goto enomem;
                return edit;

        case assoc_array_walk_found_wrong_shortcut:
                /* We found a shortcut that didn't match our key in a slot we
                 * needed to follow.
                 */
                if (!assoc_array_insert_mid_shortcut(edit, ops, &result))
                        goto enomem;
                return edit;
        }

enomem:
        /* Clean up after an out of memory error */
        pr_devel("enomem\n");
        assoc_array_cancel_edit(edit);
        return ERR_PTR(-ENOMEM);
}

/**
 * assoc_array_insert_set_object - Set the new object pointer in an edit script
 * @edit: The edit script to modify.
 * @object: The object pointer to set.
 *
 * Change the object to be inserted in an edit script.  The object pointed to
 * by the old object is not freed.  This must be done prior to applying the
 * script.
 */
void assoc_array_insert_set_object(struct assoc_array_edit *edit, void *object)
{
        BUG_ON(!object);
        edit->leaf = assoc_array_leaf_to_ptr(object);
}

struct assoc_array_delete_collapse_context {
        struct assoc_array_node        *node;
        const void                *skip_leaf;
        int                        slot;
};

/*
 * Subtree collapse to node iterator.
 */
static int assoc_array_delete_collapse_iterator(const void *leaf,
                                                void *iterator_data)
{
        struct assoc_array_delete_collapse_context *collapse = iterator_data;

        if (leaf == collapse->skip_leaf)
                return 0;

        BUG_ON(collapse->slot >= ASSOC_ARRAY_FAN_OUT);

        collapse->node->slots[collapse->slot++] = assoc_array_leaf_to_ptr(leaf);
        return 0;
}

/**
 * assoc_array_delete - Script deletion of an object from an associative array
 * @array: The array to search.
 * @ops: The operations to use.
 * @index_key: The key to the object.
 *
 * Precalculate and preallocate a script for the deletion of an object from an
 * associative array.  This results in an edit script that can either be
 * applied or cancelled.
 *
 * The function returns a pointer to an edit script if the object was found,
 * NULL if the object was not found or -ENOMEM.
 *
 * The caller should lock against other modifications and must continue to hold
 * the lock until assoc_array_apply_edit() has been called.
 *
 * Accesses to the tree may take place concurrently with this function,
 * provided they hold the RCU read lock.
 */
struct assoc_array_edit *assoc_array_delete(struct assoc_array *array,
                                            const struct assoc_array_ops *ops,
                                            const void *index_key)
{
        struct assoc_array_delete_collapse_context collapse;
        struct assoc_array_walk_result result;
        struct assoc_array_node *node, *new_n0;
        struct assoc_array_edit *edit;
        struct assoc_array_ptr *ptr;
        bool has_meta;
        int slot, i;

        pr_devel("-->%s()\n", __func__);

        edit = kzalloc(sizeof(struct assoc_array_edit), GFP_KERNEL);
        if (!edit)
                return ERR_PTR(-ENOMEM);
        edit->array = array;
        edit->ops = ops;
        edit->adjust_count_by = -1;

        switch (assoc_array_walk(array, ops, index_key, &result)) {
        case assoc_array_walk_found_terminal_node:
                /* We found a node that should contain the leaf we've been
                 * asked to remove - *if* it's in the tree.
                 */
                pr_devel("terminal_node\n");
                node = result.terminal_node.node;

                for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
                        ptr = node->slots[slot];
                        if (ptr &&
                            assoc_array_ptr_is_leaf(ptr) &&
                            ops->compare_object(assoc_array_ptr_to_leaf(ptr),
                                                index_key))
                                goto found_leaf;
                }
                fallthrough;
        case assoc_array_walk_tree_empty:
        case assoc_array_walk_found_wrong_shortcut:
        default:
                assoc_array_cancel_edit(edit);
                pr_devel("not found\n");
                return NULL;
        }

found_leaf:
        BUG_ON(array->nr_leaves_on_tree <= 0);

        /* In the simplest form of deletion we just clear the slot and release
         * the leaf after a suitable interval.
         */
        edit->dead_leaf = node->slots[slot];
        edit->set[0].ptr = &node->slots[slot];
        edit->set[0].to = NULL;
        edit->adjust_count_on = node;

        /* If that concludes erasure of the last leaf, then delete the entire
         * internal array.
         */
        if (array->nr_leaves_on_tree == 1) {
                edit->set[1].ptr = &array->root;
                edit->set[1].to = NULL;
                edit->adjust_count_on = NULL;
                edit->excised_subtree = array->root;
                pr_devel("all gone\n");
                return edit;
        }

        /* However, we'd also like to clear up some metadata blocks if we
         * possibly can.
         *
         * We go for a simple algorithm of: if this node has FAN_OUT or fewer
         * leaves in it, then attempt to collapse it - and attempt to
         * recursively collapse up the tree.
         *
         * We could also try and collapse in partially filled subtrees to take
         * up space in this node.
         */
        if (node->nr_leaves_on_branch <= ASSOC_ARRAY_FAN_OUT + 1) {
                struct assoc_array_node *parent, *grandparent;
                struct assoc_array_ptr *ptr;

                /* First of all, we need to know if this node has metadata so
                 * that we don't try collapsing if all the leaves are already
                 * here.
                 */
                has_meta = false;
                for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
                        ptr = node->slots[i];
                        if (assoc_array_ptr_is_meta(ptr)) {
                                has_meta = true;
                                break;
                        }
                }

                pr_devel("leaves: %ld [m=%d]\n",
                         node->nr_leaves_on_branch - 1, has_meta);

                /* Look further up the tree to see if we can collapse this node
                 * into a more proximal node too.
                 */
                parent = node;
        collapse_up:
                pr_devel("collapse subtree: %ld\n", parent->nr_leaves_on_branch);

                ptr = parent->back_pointer;
                if (!ptr)
                        goto do_collapse;
                if (assoc_array_ptr_is_shortcut(ptr)) {
                        struct assoc_array_shortcut *s = assoc_array_ptr_to_shortcut(ptr);
                        ptr = s->back_pointer;
                        if (!ptr)
                                goto do_collapse;
                }

                grandparent = assoc_array_ptr_to_node(ptr);
                if (grandparent->nr_leaves_on_branch <= ASSOC_ARRAY_FAN_OUT + 1) {
                        parent = grandparent;
                        goto collapse_up;
                }

        do_collapse:
                /* There's no point collapsing if the original node has no meta
                 * pointers to discard and if we didn't merge into one of that
                 * node's ancestry.
                 */
                if (has_meta || parent != node) {
                        node = parent;

                        /* Create a new node to collapse into */
                        new_n0 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
                        if (!new_n0)
                                goto enomem;
                        edit->new_meta[0] = assoc_array_node_to_ptr(new_n0);

                        new_n0->back_pointer = node->back_pointer;
                        new_n0->parent_slot = node->parent_slot;
                        new_n0->nr_leaves_on_branch = node->nr_leaves_on_branch;
                        edit->adjust_count_on = new_n0;

                        collapse.node = new_n0;
                        collapse.skip_leaf = assoc_array_ptr_to_leaf(edit->dead_leaf);
                        collapse.slot = 0;
                        assoc_array_subtree_iterate(assoc_array_node_to_ptr(node),
                                                    node->back_pointer,
                                                    assoc_array_delete_collapse_iterator,
                                                    &collapse);
                        pr_devel("collapsed %d,%lu\n", collapse.slot, new_n0->nr_leaves_on_branch);
                        BUG_ON(collapse.slot != new_n0->nr_leaves_on_branch - 1);

                        if (!node->back_pointer) {
                                edit->set[1].ptr = &array->root;
                        } else if (assoc_array_ptr_is_leaf(node->back_pointer)) {
                                BUG();
                        } else if (assoc_array_ptr_is_node(node->back_pointer)) {
                                struct assoc_array_node *p =
                                        assoc_array_ptr_to_node(node->back_pointer);
                                edit->set[1].ptr = &p->slots[node->parent_slot];
                        } else if (assoc_array_ptr_is_shortcut(node->back_pointer)) {
                                struct assoc_array_shortcut *s =
                                        assoc_array_ptr_to_shortcut(node->back_pointer);
                                edit->set[1].ptr = &s->next_node;
                        }
                        edit->set[1].to = assoc_array_node_to_ptr(new_n0);
                        edit->excised_subtree = assoc_array_node_to_ptr(node);
                }
        }

        return edit;

enomem:
        /* Clean up after an out of memory error */
        pr_devel("enomem\n");
        assoc_array_cancel_edit(edit);
        return ERR_PTR(-ENOMEM);
}

/**
 * assoc_array_clear - Script deletion of all objects from an associative array
 * @array: The array to clear.
 * @ops: The operations to use.
 *
 * Precalculate and preallocate a script for the deletion of all the objects
 * from an associative array.  This results in an edit script that can either
 * be applied or cancelled.
 *
 * The function returns a pointer to an edit script if there are objects to be
 * deleted, NULL if there are no objects in the array or -ENOMEM.
 *
 * The caller should lock against other modifications and must continue to hold
 * the lock until assoc_array_apply_edit() has been called.
 *
 * Accesses to the tree may take place concurrently with this function,
 * provided they hold the RCU read lock.
 */
struct assoc_array_edit *assoc_array_clear(struct assoc_array *array,
                                           const struct assoc_array_ops *ops)
{
        struct assoc_array_edit *edit;

        pr_devel("-->%s()\n", __func__);

        if (!array->root)
                return NULL;

        edit = kzalloc(sizeof(struct assoc_array_edit), GFP_KERNEL);
        if (!edit)
                return ERR_PTR(-ENOMEM);
        edit->array = array;
        edit->ops = ops;
        edit->set[1].ptr = &array->root;
        edit->set[1].to = NULL;
        edit->excised_subtree = array->root;
        edit->ops_for_excised_subtree = ops;
        pr_devel("all gone\n");
        return edit;
}

/*
 * Handle the deferred destruction after an applied edit.
 */
static void assoc_array_rcu_cleanup(struct rcu_head *head)
{
        struct assoc_array_edit *edit =
                container_of(head, struct assoc_array_edit, rcu);
        int i;

        pr_devel("-->%s()\n", __func__);

        if (edit->dead_leaf)
                edit->ops->free_object(assoc_array_ptr_to_leaf(edit->dead_leaf));
        for (i = 0; i < ARRAY_SIZE(edit->excised_meta); i++)
                if (edit->excised_meta[i])
                        kfree(assoc_array_ptr_to_node(edit->excised_meta[i]));

        if (edit->excised_subtree) {
                BUG_ON(assoc_array_ptr_is_leaf(edit->excised_subtree));
                if (assoc_array_ptr_is_node(edit->excised_subtree)) {
                        struct assoc_array_node *n =
                                assoc_array_ptr_to_node(edit->excised_subtree);
                        n->back_pointer = NULL;
                } else {
                        struct assoc_array_shortcut *s =
                                assoc_array_ptr_to_shortcut(edit->excised_subtree);
                        s->back_pointer = NULL;
                }
                assoc_array_destroy_subtree(edit->excised_subtree,
                                            edit->ops_for_excised_subtree);
        }

        kfree(edit);
}

/**
 * assoc_array_apply_edit - Apply an edit script to an associative array
 * @edit: The script to apply.
 *
 * Apply an edit script to an associative array to effect an insertion,
 * deletion or clearance.  As the edit script includes preallocated memory,
 * this is guaranteed not to fail.
 *
 * The edit script, dead objects and dead metadata will be scheduled for
 * destruction after an RCU grace period to permit those doing read-only
 * accesses on the array to continue to do so under the RCU read lock whilst
 * the edit is taking place.
 */
void assoc_array_apply_edit(struct assoc_array_edit *edit)
{
        struct assoc_array_shortcut *shortcut;
        struct assoc_array_node *node;
        struct assoc_array_ptr *ptr;
        int i;

        pr_devel("-->%s()\n", __func__);

        smp_wmb();
        if (edit->leaf_p)
                *edit->leaf_p = edit->leaf;

        smp_wmb();
        for (i = 0; i < ARRAY_SIZE(edit->set_parent_slot); i++)
                if (edit->set_parent_slot[i].p)
                        *edit->set_parent_slot[i].p = edit->set_parent_slot[i].to;

        smp_wmb();
        for (i = 0; i < ARRAY_SIZE(edit->set_backpointers); i++)
                if (edit->set_backpointers[i])
                        *edit->set_backpointers[i] = edit->set_backpointers_to;

        smp_wmb();
        for (i = 0; i < ARRAY_SIZE(edit->set); i++)
                if (edit->set[i].ptr)
                        *edit->set[i].ptr = edit->set[i].to;

        if (edit->array->root == NULL) {
                edit->array->nr_leaves_on_tree = 0;
        } else if (edit->adjust_count_on) {
                node = edit->adjust_count_on;
                for (;;) {
                        node->nr_leaves_on_branch += edit->adjust_count_by;

                        ptr = node->back_pointer;
                        if (!ptr)
                                break;
                        if (assoc_array_ptr_is_shortcut(ptr)) {
                                shortcut = assoc_array_ptr_to_shortcut(ptr);
                                ptr = shortcut->back_pointer;
                                if (!ptr)
                                        break;
                        }
                        BUG_ON(!assoc_array_ptr_is_node(ptr));
                        node = assoc_array_ptr_to_node(ptr);
                }

                edit->array->nr_leaves_on_tree += edit->adjust_count_by;
        }

        call_rcu(&edit->rcu, assoc_array_rcu_cleanup);
}

/**
 * assoc_array_cancel_edit - Discard an edit script.
 * @edit: The script to discard.
 *
 * Free an edit script and all the preallocated data it holds without making
 * any changes to the associative array it was intended for.
 *
 * NOTE!  In the case of an insertion script, this does _not_ release the leaf
 * that was to be inserted.  That is left to the caller.
 */
void assoc_array_cancel_edit(struct assoc_array_edit *edit)
{
        struct assoc_array_ptr *ptr;
        int i;

        pr_devel("-->%s()\n", __func__);

        /* Clean up after an out of memory error */
        for (i = 0; i < ARRAY_SIZE(edit->new_meta); i++) {
                ptr = edit->new_meta[i];
                if (ptr) {
                        if (assoc_array_ptr_is_node(ptr))
                                kfree(assoc_array_ptr_to_node(ptr));
                        else
                                kfree(assoc_array_ptr_to_shortcut(ptr));
                }
        }
        kfree(edit);
}

/**
 * assoc_array_gc - Garbage collect an associative array.
 * @array: The array to clean.
 * @ops: The operations to use.
 * @iterator: A callback function to pass judgement on each object.
 * @iterator_data: Private data for the callback function.
 *
 * Collect garbage from an associative array and pack down the internal tree to
 * save memory.
 *
 * The iterator function is asked to pass judgement upon each object in the
 * array.  If it returns false, the object is discard and if it returns true,
 * the object is kept.  If it returns true, it must increment the object's
 * usage count (or whatever it needs to do to retain it) before returning.
 *
 * This function returns 0 if successful or -ENOMEM if out of memory.  In the
 * latter case, the array is not changed.
 *
 * The caller should lock against other modifications and must continue to hold
 * the lock until assoc_array_apply_edit() has been called.
 *
 * Accesses to the tree may take place concurrently with this function,
 * provided they hold the RCU read lock.
 */
int assoc_array_gc(struct assoc_array *array,
                   const struct assoc_array_ops *ops,
                   bool (*iterator)(void *object, void *iterator_data),
                   void *iterator_data)
{
        struct assoc_array_shortcut *shortcut, *new_s;
        struct assoc_array_node *node, *new_n;
        struct assoc_array_edit *edit;
        struct assoc_array_ptr *cursor, *ptr;
        struct assoc_array_ptr *new_root, *new_parent, **new_ptr_pp;
        unsigned long nr_leaves_on_tree;
        bool retained;
        int keylen, slot, nr_free, next_slot, i;

        pr_devel("-->%s()\n", __func__);

        if (!array->root)
                return 0;

        edit = kzalloc(sizeof(struct assoc_array_edit), GFP_KERNEL);
        if (!edit)
                return -ENOMEM;
        edit->array = array;
        edit->ops = ops;
        edit->ops_for_excised_subtree = ops;
        edit->set[0].ptr = &array->root;
        edit->excised_subtree = array->root;

        new_root = new_parent = NULL;
        new_ptr_pp = &new_root;
        cursor = array->root;

descend:
        /* If this point is a shortcut, then we need to duplicate it and
         * advance the target cursor.
         */
        if (assoc_array_ptr_is_shortcut(cursor)) {
                shortcut = assoc_array_ptr_to_shortcut(cursor);
                keylen = round_up(shortcut->skip_to_level, ASSOC_ARRAY_KEY_CHUNK_SIZE);
                keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT;
                new_s = kmalloc(struct_size(new_s, index_key, keylen),
                                GFP_KERNEL);
                if (!new_s)
                        goto enomem;
                pr_devel("dup shortcut %p -> %p\n", shortcut, new_s);
                memcpy(new_s, shortcut, struct_size(new_s, index_key, keylen));
                new_s->back_pointer = new_parent;
                new_s->parent_slot = shortcut->parent_slot;
                *new_ptr_pp = new_parent = assoc_array_shortcut_to_ptr(new_s);
                new_ptr_pp = &new_s->next_node;
                cursor = shortcut->next_node;
        }

        /* Duplicate the node at this position */
        node = assoc_array_ptr_to_node(cursor);
        new_n = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
        if (!new_n)
                goto enomem;
        pr_devel("dup node %p -> %p\n", node, new_n);
        new_n->back_pointer = new_parent;
        new_n->parent_slot = node->parent_slot;
        *new_ptr_pp = new_parent = assoc_array_node_to_ptr(new_n);
        new_ptr_pp = NULL;
        slot = 0;

continue_node:
        /* Filter across any leaves and gc any subtrees */
        for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
                ptr = node->slots[slot];
                if (!ptr)
                        continue;

                if (assoc_array_ptr_is_leaf(ptr)) {
                        if (iterator(assoc_array_ptr_to_leaf(ptr),
                                     iterator_data))
                                /* The iterator will have done any reference
                                 * counting on the object for us.
                                 */
                                new_n->slots[slot] = ptr;
                        continue;
                }

                new_ptr_pp = &new_n->slots[slot];
                cursor = ptr;
                goto descend;
        }

retry_compress:
        pr_devel("-- compress node %p --\n", new_n);

        /* Count up the number of empty slots in this node and work out the
         * subtree leaf count.
         */
        new_n->nr_leaves_on_branch = 0;
        nr_free = 0;
        for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
                ptr = new_n->slots[slot];
                if (!ptr)
                        nr_free++;
                else if (assoc_array_ptr_is_leaf(ptr))
                        new_n->nr_leaves_on_branch++;
        }
        pr_devel("free=%d, leaves=%lu\n", nr_free, new_n->nr_leaves_on_branch);

        /* See what we can fold in */
        retained = false;
        next_slot = 0;
        for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
                struct assoc_array_shortcut *s;
                struct assoc_array_node *child;

                ptr = new_n->slots[slot];
                if (!ptr || assoc_array_ptr_is_leaf(ptr))
                        continue;

                s = NULL;
                if (assoc_array_ptr_is_shortcut(ptr)) {
                        s = assoc_array_ptr_to_shortcut(ptr);
                        ptr = s->next_node;
                }

                child = assoc_array_ptr_to_node(ptr);
                new_n->nr_leaves_on_branch += child->nr_leaves_on_branch;

                if (child->nr_leaves_on_branch <= nr_free + 1) {
                        /* Fold the child node into this one */
                        pr_devel("[%d] fold node %lu/%d [nx %d]\n",
                                 slot, child->nr_leaves_on_branch, nr_free + 1,
                                 next_slot);

                        /* We would already have reaped an intervening shortcut
                         * on the way back up the tree.
                         */
                        BUG_ON(s);

                        new_n->slots[slot] = NULL;
                        nr_free++;
                        if (slot < next_slot)
                                next_slot = slot;
                        for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
                                struct assoc_array_ptr *p = child->slots[i];
                                if (!p)
                                        continue;
                                BUG_ON(assoc_array_ptr_is_meta(p));
                                while (new_n->slots[next_slot])
                                        next_slot++;
                                BUG_ON(next_slot >= ASSOC_ARRAY_FAN_OUT);
                                new_n->slots[next_slot++] = p;
                                nr_free--;
                        }
                        kfree(child);
                } else {
                        pr_devel("[%d] retain node %lu/%d [nx %d]\n",
                                 slot, child->nr_leaves_on_branch, nr_free + 1,
                                 next_slot);
                        retained = true;
                }
        }

        if (retained && new_n->nr_leaves_on_branch <= ASSOC_ARRAY_FAN_OUT) {
                pr_devel("internal nodes remain despite enough space, retrying\n");
                goto retry_compress;
        }
        pr_devel("after: %lu\n", new_n->nr_leaves_on_branch);

        nr_leaves_on_tree = new_n->nr_leaves_on_branch;

        /* Excise this node if it is singly occupied by a shortcut */
        if (nr_free == ASSOC_ARRAY_FAN_OUT - 1) {
                for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++)
                        if ((ptr = new_n->slots[slot]))
                                break;

                if (assoc_array_ptr_is_meta(ptr) &&
                    assoc_array_ptr_is_shortcut(ptr)) {
                        pr_devel("excise node %p with 1 shortcut\n", new_n);
                        new_s = assoc_array_ptr_to_shortcut(ptr);
                        new_parent = new_n->back_pointer;
                        slot = new_n->parent_slot;
                        kfree(new_n);
                        if (!new_parent) {
                                new_s->back_pointer = NULL;
                                new_s->parent_slot = 0;
                                new_root = ptr;
                                goto gc_complete;
                        }

                        if (assoc_array_ptr_is_shortcut(new_parent)) {
                                /* We can discard any preceding shortcut also */
                                struct assoc_array_shortcut *s =
                                        assoc_array_ptr_to_shortcut(new_parent);

                                pr_devel("excise preceding shortcut\n");

                                new_parent = new_s->back_pointer = s->back_pointer;
                                slot = new_s->parent_slot = s->parent_slot;
                                kfree(s);
                                if (!new_parent) {
                                        new_s->back_pointer = NULL;
                                        new_s->parent_slot = 0;
                                        new_root = ptr;
                                        goto gc_complete;
                                }
                        }

                        new_s->back_pointer = new_parent;
                        new_s->parent_slot = slot;
                        new_n = assoc_array_ptr_to_node(new_parent);
                        new_n->slots[slot] = ptr;
                        goto ascend_old_tree;
                }
        }

        /* Excise any shortcuts we might encounter that point to nodes that
         * only contain leaves.
         */
        ptr = new_n->back_pointer;
        if (!ptr)
                goto gc_complete;

        if (assoc_array_ptr_is_shortcut(ptr)) {
                new_s = assoc_array_ptr_to_shortcut(ptr);
                new_parent = new_s->back_pointer;
                slot = new_s->parent_slot;

                if (new_n->nr_leaves_on_branch <= ASSOC_ARRAY_FAN_OUT) {
                        struct assoc_array_node *n;

                        pr_devel("excise shortcut\n");
                        new_n->back_pointer = new_parent;
                        new_n->parent_slot = slot;
                        kfree(new_s);
                        if (!new_parent) {
                                new_root = assoc_array_node_to_ptr(new_n);
                                goto gc_complete;
                        }

                        n = assoc_array_ptr_to_node(new_parent);
                        n->slots[slot] = assoc_array_node_to_ptr(new_n);
                }
        } else {
                new_parent = ptr;
        }
        new_n = assoc_array_ptr_to_node(new_parent);

ascend_old_tree:
        ptr = node->back_pointer;
        if (assoc_array_ptr_is_shortcut(ptr)) {
                shortcut = assoc_array_ptr_to_shortcut(ptr);
                slot = shortcut->parent_slot;
                cursor = shortcut->back_pointer;
                if (!cursor)
                        goto gc_complete;
        } else {
                slot = node->parent_slot;
                cursor = ptr;
        }
        BUG_ON(!cursor);
        node = assoc_array_ptr_to_node(cursor);
        slot++;
        goto continue_node;

gc_complete:
        edit->set[0].to = new_root;
        assoc_array_apply_edit(edit);
        array->nr_leaves_on_tree = nr_leaves_on_tree;
        return 0;

enomem:
        pr_devel("enomem\n");
        assoc_array_destroy_subtree(new_root, edit->ops);
        kfree(edit);
        return -ENOMEM;
}





















































































































































































    3 










































































































    3 
    3 


















































































































































    3 
    3 















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    5 









    1 







    4 




    3 

















































































































































































    1 






    1 



























    6 









    2 







    1 



    3 















    3 








    3 










































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Network block device - make block devices work over TCP
 *
 * Note that you can not swap over this thing, yet. Seems to work but
 * deadlocks sometimes - you can not swap over TCP in general.
 * 
 * Copyright 1997-2000, 2008 Pavel Machek <pavel@ucw.cz>
 * Parts copyright 2001 Steven Whitehouse <steve@chygwyn.com>
 *
 * (part of code stolen from loop.c)
 */

#define pr_fmt(fmt) "nbd: " fmt

#include <linux/major.h>

#include <linux/blkdev.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/fs.h>
#include <linux/bio.h>
#include <linux/stat.h>
#include <linux/errno.h>
#include <linux/file.h>
#include <linux/ioctl.h>
#include <linux/mutex.h>
#include <linux/compiler.h>
#include <linux/completion.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <net/sock.h>
#include <linux/net.h>
#include <linux/kthread.h>
#include <linux/types.h>
#include <linux/debugfs.h>
#include <linux/blk-mq.h>

#include <linux/uaccess.h>
#include <asm/types.h>

#include <linux/nbd.h>
#include <linux/nbd-netlink.h>
#include <net/genetlink.h>

#define CREATE_TRACE_POINTS
#include <trace/events/nbd.h>

static DEFINE_IDR(nbd_index_idr);
static DEFINE_MUTEX(nbd_index_mutex);
static struct workqueue_struct *nbd_del_wq;
static int nbd_total_devices = 0;

struct nbd_sock {
        struct socket *sock;
        struct mutex tx_lock;
        struct request *pending;
        int sent;
        bool dead;
        int fallback_index;
        int cookie;
        struct work_struct work;
};

struct recv_thread_args {
        struct work_struct work;
        struct nbd_device *nbd;
        struct nbd_sock *nsock;
        int index;
};

struct link_dead_args {
        struct work_struct work;
        int index;
};

#define NBD_RT_TIMEDOUT                        0
#define NBD_RT_DISCONNECT_REQUESTED        1
#define NBD_RT_DISCONNECTED                2
#define NBD_RT_HAS_PID_FILE                3
#define NBD_RT_HAS_CONFIG_REF                4
#define NBD_RT_BOUND                        5
#define NBD_RT_DISCONNECT_ON_CLOSE        6
#define NBD_RT_HAS_BACKEND_FILE                7

#define NBD_DESTROY_ON_DISCONNECT        0
#define NBD_DISCONNECT_REQUESTED        1

struct nbd_config {
        u32 flags;
        unsigned long runtime_flags;
        u64 dead_conn_timeout;

        struct nbd_sock **socks;
        int num_connections;
        atomic_t live_connections;
        wait_queue_head_t conn_wait;

        atomic_t recv_threads;
        wait_queue_head_t recv_wq;
        unsigned int blksize_bits;
        loff_t bytesize;
#if IS_ENABLED(CONFIG_DEBUG_FS)
        struct dentry *dbg_dir;
#endif
};

static inline unsigned int nbd_blksize(struct nbd_config *config)
{
        return 1u << config->blksize_bits;
}

struct nbd_device {
        struct blk_mq_tag_set tag_set;

        int index;
        refcount_t config_refs;
        refcount_t refs;
        struct nbd_config *config;
        struct mutex config_lock;
        struct gendisk *disk;
        struct workqueue_struct *recv_workq;
        struct work_struct remove_work;

        struct list_head list;
        struct task_struct *task_setup;

        unsigned long flags;
        pid_t pid; /* pid of nbd-client, if attached */

        char *backend;
};

#define NBD_CMD_REQUEUED        1
/*
 * This flag will be set if nbd_queue_rq() succeed, and will be checked and
 * cleared in completion. Both setting and clearing of the flag are protected
 * by cmd->lock.
 */
#define NBD_CMD_INFLIGHT        2

/* Just part of request header or data payload is sent successfully */
#define NBD_CMD_PARTIAL_SEND        3

struct nbd_cmd {
        struct nbd_device *nbd;
        struct mutex lock;
        int index;
        int cookie;
        int retries;
        blk_status_t status;
        unsigned long flags;
        u32 cmd_cookie;
};

#if IS_ENABLED(CONFIG_DEBUG_FS)
static struct dentry *nbd_dbg_dir;
#endif

#define nbd_name(nbd) ((nbd)->disk->disk_name)

#define NBD_DEF_BLKSIZE_BITS 10

static unsigned int nbds_max = 16;
static int max_part = 16;
static int part_shift;

static int nbd_dev_dbg_init(struct nbd_device *nbd);
static void nbd_dev_dbg_close(struct nbd_device *nbd);
static void nbd_config_put(struct nbd_device *nbd);
static void nbd_connect_reply(struct genl_info *info, int index);
static int nbd_genl_status(struct sk_buff *skb, struct genl_info *info);
static void nbd_dead_link_work(struct work_struct *work);
static void nbd_disconnect_and_put(struct nbd_device *nbd);

static inline struct device *nbd_to_dev(struct nbd_device *nbd)
{
        return disk_to_dev(nbd->disk);
}

static void nbd_requeue_cmd(struct nbd_cmd *cmd)
{
        struct request *req = blk_mq_rq_from_pdu(cmd);

        lockdep_assert_held(&cmd->lock);

        /*
         * Clear INFLIGHT flag so that this cmd won't be completed in
         * normal completion path
         *
         * INFLIGHT flag will be set when the cmd is queued to nbd next
         * time.
         */
        __clear_bit(NBD_CMD_INFLIGHT, &cmd->flags);

        if (!test_and_set_bit(NBD_CMD_REQUEUED, &cmd->flags))
                blk_mq_requeue_request(req, true);
}

#define NBD_COOKIE_BITS 32

static u64 nbd_cmd_handle(struct nbd_cmd *cmd)
{
        struct request *req = blk_mq_rq_from_pdu(cmd);
        u32 tag = blk_mq_unique_tag(req);
        u64 cookie = cmd->cmd_cookie;

        return (cookie << NBD_COOKIE_BITS) | tag;
}

static u32 nbd_handle_to_tag(u64 handle)
{
        return (u32)handle;
}

static u32 nbd_handle_to_cookie(u64 handle)
{
        return (u32)(handle >> NBD_COOKIE_BITS);
}

static const char *nbdcmd_to_ascii(int cmd)
{
        switch (cmd) {
        case  NBD_CMD_READ: return "read";
        case NBD_CMD_WRITE: return "write";
        case  NBD_CMD_DISC: return "disconnect";
        case NBD_CMD_FLUSH: return "flush";
        case  NBD_CMD_TRIM: return "trim/discard";
        }
        return "invalid";
}

static ssize_t pid_show(struct device *dev,
                        struct device_attribute *attr, char *buf)
{
        struct gendisk *disk = dev_to_disk(dev);
        struct nbd_device *nbd = disk->private_data;

        return sprintf(buf, "%d\n", nbd->pid);
}

static const struct device_attribute pid_attr = {
        .attr = { .name = "pid", .mode = 0444},
        .show = pid_show,
};

static ssize_t backend_show(struct device *dev,
                struct device_attribute *attr, char *buf)
{
        struct gendisk *disk = dev_to_disk(dev);
        struct nbd_device *nbd = disk->private_data;

        return sprintf(buf, "%s\n", nbd->backend ?: "");
}

static const struct device_attribute backend_attr = {
        .attr = { .name = "backend", .mode = 0444},
        .show = backend_show,
};

static void nbd_dev_remove(struct nbd_device *nbd)
{
        struct gendisk *disk = nbd->disk;

        del_gendisk(disk);
        blk_mq_free_tag_set(&nbd->tag_set);

        /*
         * Remove from idr after del_gendisk() completes, so if the same ID is
         * reused, the following add_disk() will succeed.
         */
        mutex_lock(&nbd_index_mutex);
        idr_remove(&nbd_index_idr, nbd->index);
        mutex_unlock(&nbd_index_mutex);
        destroy_workqueue(nbd->recv_workq);
        put_disk(disk);
}

static void nbd_dev_remove_work(struct work_struct *work)
{
        nbd_dev_remove(container_of(work, struct nbd_device, remove_work));
}

static void nbd_put(struct nbd_device *nbd)
{
        if (!refcount_dec_and_test(&nbd->refs))
                return;

        /* Call del_gendisk() asynchrounously to prevent deadlock */
        if (test_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags))
                queue_work(nbd_del_wq, &nbd->remove_work);
        else
                nbd_dev_remove(nbd);
}

static int nbd_disconnected(struct nbd_config *config)
{
        return test_bit(NBD_RT_DISCONNECTED, &config->runtime_flags) ||
                test_bit(NBD_RT_DISCONNECT_REQUESTED, &config->runtime_flags);
}

static void nbd_mark_nsock_dead(struct nbd_device *nbd, struct nbd_sock *nsock,
                                int notify)
{
        if (!nsock->dead && notify && !nbd_disconnected(nbd->config)) {
                struct link_dead_args *args;
                args = kmalloc(sizeof(struct link_dead_args), GFP_NOIO);
                if (args) {
                        INIT_WORK(&args->work, nbd_dead_link_work);
                        args->index = nbd->index;
                        queue_work(system_wq, &args->work);
                }
        }
        if (!nsock->dead) {
                kernel_sock_shutdown(nsock->sock, SHUT_RDWR);
                if (atomic_dec_return(&nbd->config->live_connections) == 0) {
                        if (test_and_clear_bit(NBD_RT_DISCONNECT_REQUESTED,
                                               &nbd->config->runtime_flags)) {
                                set_bit(NBD_RT_DISCONNECTED,
                                        &nbd->config->runtime_flags);
                                dev_info(nbd_to_dev(nbd),
                                        "Disconnected due to user request.\n");
                        }
                }
        }
        nsock->dead = true;
        nsock->pending = NULL;
        nsock->sent = 0;
}

static int nbd_set_size(struct nbd_device *nbd, loff_t bytesize, loff_t blksize)
{
        struct queue_limits lim;
        int error;

        if (!blksize)
                blksize = 1u << NBD_DEF_BLKSIZE_BITS;

        if (blk_validate_block_size(blksize))
                return -EINVAL;

        if (bytesize < 0)
                return -EINVAL;

        nbd->config->bytesize = bytesize;
        nbd->config->blksize_bits = __ffs(blksize);

        if (!nbd->pid)
                return 0;

        lim = queue_limits_start_update(nbd->disk->queue);
        if (nbd->config->flags & NBD_FLAG_SEND_TRIM)
                lim.max_hw_discard_sectors = UINT_MAX >> SECTOR_SHIFT;
        else
                lim.max_hw_discard_sectors = 0;
        if (!(nbd->config->flags & NBD_FLAG_SEND_FLUSH)) {
                lim.features &= ~(BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA);
        } else if (nbd->config->flags & NBD_FLAG_SEND_FUA) {
                lim.features |= BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA;
        } else {
                lim.features |= BLK_FEAT_WRITE_CACHE;
                lim.features &= ~BLK_FEAT_FUA;
        }
        if (nbd->config->flags & NBD_FLAG_ROTATIONAL)
                lim.features |= BLK_FEAT_ROTATIONAL;
        if (nbd->config->flags & NBD_FLAG_SEND_WRITE_ZEROES)
                lim.max_write_zeroes_sectors = UINT_MAX >> SECTOR_SHIFT;

        lim.logical_block_size = blksize;
        lim.physical_block_size = blksize;
        error = queue_limits_commit_update_frozen(nbd->disk->queue, &lim);
        if (error)
                return error;

        if (max_part)
                set_bit(GD_NEED_PART_SCAN, &nbd->disk->state);
        if (!set_capacity_and_notify(nbd->disk, bytesize >> 9))
                kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE);
        return 0;
}

static void nbd_complete_rq(struct request *req)
{
        struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req);

        dev_dbg(nbd_to_dev(cmd->nbd), "request %p: %s\n", req,
                cmd->status ? "failed" : "done");

        blk_mq_end_request(req, cmd->status);
}

/*
 * Forcibly shutdown the socket causing all listeners to error
 */
static void sock_shutdown(struct nbd_device *nbd)
{
        struct nbd_config *config = nbd->config;
        int i;

        if (config->num_connections == 0)
                return;
        if (test_and_set_bit(NBD_RT_DISCONNECTED, &config->runtime_flags))
                return;

        for (i = 0; i < config->num_connections; i++) {
                struct nbd_sock *nsock = config->socks[i];
                mutex_lock(&nsock->tx_lock);
                nbd_mark_nsock_dead(nbd, nsock, 0);
                mutex_unlock(&nsock->tx_lock);
        }
        dev_warn(disk_to_dev(nbd->disk), "shutting down sockets\n");
}

static u32 req_to_nbd_cmd_type(struct request *req)
{
        switch (req_op(req)) {
        case REQ_OP_DISCARD:
                return NBD_CMD_TRIM;
        case REQ_OP_FLUSH:
                return NBD_CMD_FLUSH;
        case REQ_OP_WRITE:
                return NBD_CMD_WRITE;
        case REQ_OP_READ:
                return NBD_CMD_READ;
        case REQ_OP_WRITE_ZEROES:
                return NBD_CMD_WRITE_ZEROES;
        default:
                return U32_MAX;
        }
}

static struct nbd_config *nbd_get_config_unlocked(struct nbd_device *nbd)
{
        if (refcount_inc_not_zero(&nbd->config_refs)) {
                /*
                 * Add smp_mb__after_atomic to ensure that reading nbd->config_refs
                 * and reading nbd->config is ordered. The pair is the barrier in
                 * nbd_alloc_and_init_config(), avoid nbd->config_refs is set
                 * before nbd->config.
                 */
                smp_mb__after_atomic();
                return nbd->config;
        }

        return NULL;
}

static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req)
{
        struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req);
        struct nbd_device *nbd = cmd->nbd;
        struct nbd_config *config;

        if (!mutex_trylock(&cmd->lock))
                return BLK_EH_RESET_TIMER;

        /* partial send is handled in nbd_sock's work function */
        if (test_bit(NBD_CMD_PARTIAL_SEND, &cmd->flags)) {
                mutex_unlock(&cmd->lock);
                return BLK_EH_RESET_TIMER;
        }

        if (!test_bit(NBD_CMD_INFLIGHT, &cmd->flags)) {
                mutex_unlock(&cmd->lock);
                return BLK_EH_DONE;
        }

        config = nbd_get_config_unlocked(nbd);
        if (!config) {
                cmd->status = BLK_STS_TIMEOUT;
                __clear_bit(NBD_CMD_INFLIGHT, &cmd->flags);
                mutex_unlock(&cmd->lock);
                goto done;
        }

        if (config->num_connections > 1 ||
            (config->num_connections == 1 && nbd->tag_set.timeout)) {
                dev_err_ratelimited(nbd_to_dev(nbd),
                                    "Connection timed out, retrying (%d/%d alive)\n",
                                    atomic_read(&config->live_connections),
                                    config->num_connections);
                /*
                 * Hooray we have more connections, requeue this IO, the submit
                 * path will put it on a real connection. Or if only one
                 * connection is configured, the submit path will wait util
                 * a new connection is reconfigured or util dead timeout.
                 */
                if (config->socks) {
                        if (cmd->index < config->num_connections) {
                                struct nbd_sock *nsock =
                                        config->socks[cmd->index];
                                mutex_lock(&nsock->tx_lock);
                                /* We can have multiple outstanding requests, so
                                 * we don't want to mark the nsock dead if we've
                                 * already reconnected with a new socket, so
                                 * only mark it dead if its the same socket we
                                 * were sent out on.
                                 */
                                if (cmd->cookie == nsock->cookie)
                                        nbd_mark_nsock_dead(nbd, nsock, 1);
                                mutex_unlock(&nsock->tx_lock);
                        }
                        nbd_requeue_cmd(cmd);
                        mutex_unlock(&cmd->lock);
                        nbd_config_put(nbd);
                        return BLK_EH_DONE;
                }
        }

        if (!nbd->tag_set.timeout) {
                /*
                 * Userspace sets timeout=0 to disable socket disconnection,
                 * so just warn and reset the timer.
                 */
                struct nbd_sock *nsock = config->socks[cmd->index];
                cmd->retries++;
                dev_info(nbd_to_dev(nbd), "Possible stuck request %p: control (%s@%llu,%uB). Runtime %u seconds\n",
                        req, nbdcmd_to_ascii(req_to_nbd_cmd_type(req)),
                        (unsigned long long)blk_rq_pos(req) << 9,
                        blk_rq_bytes(req), (req->timeout / HZ) * cmd->retries);

                mutex_lock(&nsock->tx_lock);
                if (cmd->cookie != nsock->cookie) {
                        nbd_requeue_cmd(cmd);
                        mutex_unlock(&nsock->tx_lock);
                        mutex_unlock(&cmd->lock);
                        nbd_config_put(nbd);
                        return BLK_EH_DONE;
                }
                mutex_unlock(&nsock->tx_lock);
                mutex_unlock(&cmd->lock);
                nbd_config_put(nbd);
                return BLK_EH_RESET_TIMER;
        }

        dev_err_ratelimited(nbd_to_dev(nbd), "Connection timed out\n");
        set_bit(NBD_RT_TIMEDOUT, &config->runtime_flags);
        cmd->status = BLK_STS_IOERR;
        __clear_bit(NBD_CMD_INFLIGHT, &cmd->flags);
        mutex_unlock(&cmd->lock);
        sock_shutdown(nbd);
        nbd_config_put(nbd);
done:
        blk_mq_complete_request(req);
        return BLK_EH_DONE;
}

static int __sock_xmit(struct nbd_device *nbd, struct socket *sock, int send,
                       struct iov_iter *iter, int msg_flags, int *sent)
{
        int result;
        struct msghdr msg = {} ;
        unsigned int noreclaim_flag;

        if (unlikely(!sock)) {
                dev_err_ratelimited(disk_to_dev(nbd->disk),
                        "Attempted %s on closed socket in sock_xmit\n",
                        (send ? "send" : "recv"));
                return -EINVAL;
        }

        msg.msg_iter = *iter;

        noreclaim_flag = memalloc_noreclaim_save();
        do {
                sock->sk->sk_allocation = GFP_NOIO | __GFP_MEMALLOC;
                sock->sk->sk_use_task_frag = false;
                msg.msg_flags = msg_flags | MSG_NOSIGNAL;

                if (send)
                        result = sock_sendmsg(sock, &msg);
                else
                        result = sock_recvmsg(sock, &msg, msg.msg_flags);

                if (result <= 0) {
                        if (result == 0)
                                result = -EPIPE; /* short read */
                        break;
                }
                if (sent)
                        *sent += result;
        } while (msg_data_left(&msg));

        memalloc_noreclaim_restore(noreclaim_flag);

        return result;
}

/*
 *  Send or receive packet. Return a positive value on success and
 *  negtive value on failure, and never return 0.
 */
static int sock_xmit(struct nbd_device *nbd, int index, int send,
                     struct iov_iter *iter, int msg_flags, int *sent)
{
        struct nbd_config *config = nbd->config;
        struct socket *sock = config->socks[index]->sock;

        return __sock_xmit(nbd, sock, send, iter, msg_flags, sent);
}

/*
 * Different settings for sk->sk_sndtimeo can result in different return values
 * if there is a signal pending when we enter sendmsg, because reasons?
 */
static inline int was_interrupted(int result)
{
        return result == -ERESTARTSYS || result == -EINTR;
}

/*
 * We've already sent header or part of data payload, have no choice but
 * to set pending and schedule it in work.
 *
 * And we have to return BLK_STS_OK to block core, otherwise this same
 * request may be re-dispatched with different tag, but our header has
 * been sent out with old tag, and this way does confuse reply handling.
 */
static void nbd_sched_pending_work(struct nbd_device *nbd,
                                   struct nbd_sock *nsock,
                                   struct nbd_cmd *cmd, int sent)
{
        struct request *req = blk_mq_rq_from_pdu(cmd);

        /* pending work should be scheduled only once */
        WARN_ON_ONCE(test_bit(NBD_CMD_PARTIAL_SEND, &cmd->flags));

        nsock->pending = req;
        nsock->sent = sent;
        set_bit(NBD_CMD_PARTIAL_SEND, &cmd->flags);
        refcount_inc(&nbd->config_refs);
        schedule_work(&nsock->work);
}

/*
 * Returns BLK_STS_RESOURCE if the caller should retry after a delay.
 * Returns BLK_STS_IOERR if sending failed.
 */
static blk_status_t nbd_send_cmd(struct nbd_device *nbd, struct nbd_cmd *cmd,
                                 int index)
{
        struct request *req = blk_mq_rq_from_pdu(cmd);
        struct nbd_config *config = nbd->config;
        struct nbd_sock *nsock = config->socks[index];
        int result;
        struct nbd_request request = {.magic = htonl(NBD_REQUEST_MAGIC)};
        struct kvec iov = {.iov_base = &request, .iov_len = sizeof(request)};
        struct iov_iter from;
        struct bio *bio;
        u64 handle;
        u32 type;
        u32 nbd_cmd_flags = 0;
        int sent = nsock->sent, skip = 0;

        lockdep_assert_held(&cmd->lock);
        lockdep_assert_held(&nsock->tx_lock);

        iov_iter_kvec(&from, ITER_SOURCE, &iov, 1, sizeof(request));

        type = req_to_nbd_cmd_type(req);
        if (type == U32_MAX)
                return BLK_STS_IOERR;

        if (rq_data_dir(req) == WRITE &&
            (config->flags & NBD_FLAG_READ_ONLY)) {
                dev_err_ratelimited(disk_to_dev(nbd->disk),
                                    "Write on read-only\n");
                return BLK_STS_IOERR;
        }

        if (req->cmd_flags & REQ_FUA)
                nbd_cmd_flags |= NBD_CMD_FLAG_FUA;
        if ((req->cmd_flags & REQ_NOUNMAP) && (type == NBD_CMD_WRITE_ZEROES))
                nbd_cmd_flags |= NBD_CMD_FLAG_NO_HOLE;

        /* We did a partial send previously, and we at least sent the whole
         * request struct, so just go and send the rest of the pages in the
         * request.
         */
        if (sent) {
                if (sent >= sizeof(request)) {
                        skip = sent - sizeof(request);

                        /* initialize handle for tracing purposes */
                        handle = nbd_cmd_handle(cmd);

                        goto send_pages;
                }
                iov_iter_advance(&from, sent);
        } else {
                cmd->cmd_cookie++;
        }
        cmd->index = index;
        cmd->cookie = nsock->cookie;
        cmd->retries = 0;
        request.type = htonl(type | nbd_cmd_flags);
        if (type != NBD_CMD_FLUSH) {
                request.from = cpu_to_be64((u64)blk_rq_pos(req) << 9);
                request.len = htonl(blk_rq_bytes(req));
        }
        handle = nbd_cmd_handle(cmd);
        request.cookie = cpu_to_be64(handle);

        trace_nbd_send_request(&request, nbd->index, blk_mq_rq_from_pdu(cmd));

        dev_dbg(nbd_to_dev(nbd), "request %p: sending control (%s@%llu,%uB)\n",
                req, nbdcmd_to_ascii(type),
                (unsigned long long)blk_rq_pos(req) << 9, blk_rq_bytes(req));
        result = sock_xmit(nbd, index, 1, &from,
                        (type == NBD_CMD_WRITE) ? MSG_MORE : 0, &sent);
        trace_nbd_header_sent(req, handle);
        if (result < 0) {
                if (was_interrupted(result)) {
                        /* If we haven't sent anything we can just return BUSY,
                         * however if we have sent something we need to make
                         * sure we only allow this req to be sent until we are
                         * completely done.
                         */
                        if (sent) {
                                nbd_sched_pending_work(nbd, nsock, cmd, sent);
                                return BLK_STS_OK;
                        }
                        set_bit(NBD_CMD_REQUEUED, &cmd->flags);
                        return BLK_STS_RESOURCE;
                }
                dev_err_ratelimited(disk_to_dev(nbd->disk),
                        "Send control failed (result %d)\n", result);
                goto requeue;
        }
send_pages:
        if (type != NBD_CMD_WRITE)
                goto out;

        bio = req->bio;
        while (bio) {
                struct bio *next = bio->bi_next;
                struct bvec_iter iter;
                struct bio_vec bvec;

                bio_for_each_segment(bvec, bio, iter) {
                        bool is_last = !next && bio_iter_last(bvec, iter);
                        int flags = is_last ? 0 : MSG_MORE;

                        dev_dbg(nbd_to_dev(nbd), "request %p: sending %d bytes data\n",
                                req, bvec.bv_len);
                        iov_iter_bvec(&from, ITER_SOURCE, &bvec, 1, bvec.bv_len);
                        if (skip) {
                                if (skip >= iov_iter_count(&from)) {
                                        skip -= iov_iter_count(&from);
                                        continue;
                                }
                                iov_iter_advance(&from, skip);
                                skip = 0;
                        }
                        result = sock_xmit(nbd, index, 1, &from, flags, &sent);
                        if (result < 0) {
                                if (was_interrupted(result)) {
                                        nbd_sched_pending_work(nbd, nsock, cmd, sent);
                                        return BLK_STS_OK;
                                }
                                dev_err(disk_to_dev(nbd->disk),
                                        "Send data failed (result %d)\n",
                                        result);
                                goto requeue;
                        }
                        /*
                         * The completion might already have come in,
                         * so break for the last one instead of letting
                         * the iterator do it. This prevents use-after-free
                         * of the bio.
                         */
                        if (is_last)
                                break;
                }
                bio = next;
        }
out:
        trace_nbd_payload_sent(req, handle);
        nsock->pending = NULL;
        nsock->sent = 0;
        __set_bit(NBD_CMD_INFLIGHT, &cmd->flags);
        return BLK_STS_OK;

requeue:
        /*
         * Can't requeue in case we are dealing with partial send
         *
         * We must run from pending work function.
         * */
        if (test_bit(NBD_CMD_PARTIAL_SEND, &cmd->flags))
                return BLK_STS_OK;

        /* retry on a different socket */
        dev_err_ratelimited(disk_to_dev(nbd->disk),
                            "Request send failed, requeueing\n");
        nbd_mark_nsock_dead(nbd, nsock, 1);
        nbd_requeue_cmd(cmd);
        return BLK_STS_OK;
}

/* handle partial sending */
static void nbd_pending_cmd_work(struct work_struct *work)
{
        struct nbd_sock *nsock = container_of(work, struct nbd_sock, work);
        struct request *req = nsock->pending;
        struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req);
        struct nbd_device *nbd = cmd->nbd;
        unsigned long deadline = READ_ONCE(req->deadline);
        unsigned int wait_ms = 2;

        mutex_lock(&cmd->lock);

        WARN_ON_ONCE(test_bit(NBD_CMD_REQUEUED, &cmd->flags));
        if (WARN_ON_ONCE(!test_bit(NBD_CMD_PARTIAL_SEND, &cmd->flags)))
                goto out;

        mutex_lock(&nsock->tx_lock);
        while (true) {
                nbd_send_cmd(nbd, cmd, cmd->index);
                if (!nsock->pending)
                        break;

                /* don't bother timeout handler for partial sending */
                if (READ_ONCE(jiffies) + msecs_to_jiffies(wait_ms) >= deadline) {
                        cmd->status = BLK_STS_IOERR;
                        blk_mq_complete_request(req);
                        break;
                }
                msleep(wait_ms);
                wait_ms *= 2;
        }
        mutex_unlock(&nsock->tx_lock);
        clear_bit(NBD_CMD_PARTIAL_SEND, &cmd->flags);
out:
        mutex_unlock(&cmd->lock);
        nbd_config_put(nbd);
}

static int nbd_read_reply(struct nbd_device *nbd, struct socket *sock,
                          struct nbd_reply *reply)
{
        struct kvec iov = {.iov_base = reply, .iov_len = sizeof(*reply)};
        struct iov_iter to;
        int result;

        reply->magic = 0;
        iov_iter_kvec(&to, ITER_DEST, &iov, 1, sizeof(*reply));
        result = __sock_xmit(nbd, sock, 0, &to, MSG_WAITALL, NULL);
        if (result < 0) {
                if (!nbd_disconnected(nbd->config))
                        dev_err(disk_to_dev(nbd->disk),
                                "Receive control failed (result %d)\n", result);
                return result;
        }

        if (ntohl(reply->magic) != NBD_REPLY_MAGIC) {
                dev_err(disk_to_dev(nbd->disk), "Wrong magic (0x%lx)\n",
                                (unsigned long)ntohl(reply->magic));
                return -EPROTO;
        }

        return 0;
}

/* NULL returned = something went wrong, inform userspace */
static struct nbd_cmd *nbd_handle_reply(struct nbd_device *nbd, int index,
                                        struct nbd_reply *reply)
{
        int result;
        struct nbd_cmd *cmd;
        struct request *req = NULL;
        u64 handle;
        u16 hwq;
        u32 tag;
        int ret = 0;

        handle = be64_to_cpu(reply->cookie);
        tag = nbd_handle_to_tag(handle);
        hwq = blk_mq_unique_tag_to_hwq(tag);
        if (hwq < nbd->tag_set.nr_hw_queues)
                req = blk_mq_tag_to_rq(nbd->tag_set.tags[hwq],
                                       blk_mq_unique_tag_to_tag(tag));
        if (!req || !blk_mq_request_started(req)) {
                dev_err(disk_to_dev(nbd->disk), "Unexpected reply (%d) %p\n",
                        tag, req);
                return ERR_PTR(-ENOENT);
        }
        trace_nbd_header_received(req, handle);
        cmd = blk_mq_rq_to_pdu(req);

        mutex_lock(&cmd->lock);
        if (!test_bit(NBD_CMD_INFLIGHT, &cmd->flags)) {
                dev_err(disk_to_dev(nbd->disk), "Suspicious reply %d (status %u flags %lu)",
                        tag, cmd->status, cmd->flags);
                ret = -ENOENT;
                goto out;
        }
        if (cmd->index != index) {
                dev_err(disk_to_dev(nbd->disk), "Unexpected reply %d from different sock %d (expected %d)",
                        tag, index, cmd->index);
                ret = -ENOENT;
                goto out;
        }
        if (cmd->cmd_cookie != nbd_handle_to_cookie(handle)) {
                dev_err(disk_to_dev(nbd->disk), "Double reply on req %p, cmd_cookie %u, handle cookie %u\n",
                        req, cmd->cmd_cookie, nbd_handle_to_cookie(handle));
                ret = -ENOENT;
                goto out;
        }
        if (cmd->status != BLK_STS_OK) {
                dev_err(disk_to_dev(nbd->disk), "Command already handled %p\n",
                        req);
                ret = -ENOENT;
                goto out;
        }
        if (test_bit(NBD_CMD_REQUEUED, &cmd->flags)) {
                dev_err(disk_to_dev(nbd->disk), "Raced with timeout on req %p\n",
                        req);
                ret = -ENOENT;
                goto out;
        }
        if (ntohl(reply->error)) {
                dev_err(disk_to_dev(nbd->disk), "Other side returned error (%d)\n",
                        ntohl(reply->error));
                cmd->status = BLK_STS_IOERR;
                goto out;
        }

        dev_dbg(nbd_to_dev(nbd), "request %p: got reply\n", req);
        if (rq_data_dir(req) != WRITE) {
                struct req_iterator iter;
                struct bio_vec bvec;
                struct iov_iter to;

                rq_for_each_segment(bvec, req, iter) {
                        iov_iter_bvec(&to, ITER_DEST, &bvec, 1, bvec.bv_len);
                        result = sock_xmit(nbd, index, 0, &to, MSG_WAITALL, NULL);
                        if (result < 0) {
                                dev_err(disk_to_dev(nbd->disk), "Receive data failed (result %d)\n",
                                        result);
                                /*
                                 * If we've disconnected, we need to make sure we
                                 * complete this request, otherwise error out
                                 * and let the timeout stuff handle resubmitting
                                 * this request onto another connection.
                                 */
                                if (nbd_disconnected(nbd->config)) {
                                        cmd->status = BLK_STS_IOERR;
                                        goto out;
                                }
                                ret = -EIO;
                                goto out;
                        }
                        dev_dbg(nbd_to_dev(nbd), "request %p: got %d bytes data\n",
                                req, bvec.bv_len);
                }
        }
out:
        trace_nbd_payload_received(req, handle);
        mutex_unlock(&cmd->lock);
        return ret ? ERR_PTR(ret) : cmd;
}

static void recv_work(struct work_struct *work)
{
        struct recv_thread_args *args = container_of(work,
                                                     struct recv_thread_args,
                                                     work);
        struct nbd_device *nbd = args->nbd;
        struct nbd_config *config = nbd->config;
        struct request_queue *q = nbd->disk->queue;
        struct nbd_sock *nsock = args->nsock;
        struct nbd_cmd *cmd;
        struct request *rq;

        while (1) {
                struct nbd_reply reply;

                if (nbd_read_reply(nbd, nsock->sock, &reply))
                        break;

                /*
                 * Grab .q_usage_counter so request pool won't go away, then no
                 * request use-after-free is possible during nbd_handle_reply().
                 * If queue is frozen, there won't be any inflight requests, we
                 * needn't to handle the incoming garbage message.
                 */
                if (!percpu_ref_tryget(&q->q_usage_counter)) {
                        dev_err(disk_to_dev(nbd->disk), "%s: no io inflight\n",
                                __func__);
                        break;
                }

                cmd = nbd_handle_reply(nbd, args->index, &reply);
                if (IS_ERR(cmd)) {
                        percpu_ref_put(&q->q_usage_counter);
                        break;
                }

                rq = blk_mq_rq_from_pdu(cmd);
                if (likely(!blk_should_fake_timeout(rq->q))) {
                        bool complete;

                        mutex_lock(&cmd->lock);
                        complete = __test_and_clear_bit(NBD_CMD_INFLIGHT,
                                                        &cmd->flags);
                        mutex_unlock(&cmd->lock);
                        if (complete)
                                blk_mq_complete_request(rq);
                }
                percpu_ref_put(&q->q_usage_counter);
        }

        mutex_lock(&nsock->tx_lock);
        nbd_mark_nsock_dead(nbd, nsock, 1);
        mutex_unlock(&nsock->tx_lock);

        nbd_config_put(nbd);
        atomic_dec(&config->recv_threads);
        wake_up(&config->recv_wq);
        kfree(args);
}

static bool nbd_clear_req(struct request *req, void *data)
{
        struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req);

        /* don't abort one completed request */
        if (blk_mq_request_completed(req))
                return true;

        mutex_lock(&cmd->lock);
        if (!__test_and_clear_bit(NBD_CMD_INFLIGHT, &cmd->flags)) {
                mutex_unlock(&cmd->lock);
                return true;
        }
        cmd->status = BLK_STS_IOERR;
        mutex_unlock(&cmd->lock);

        blk_mq_complete_request(req);
        return true;
}

static void nbd_clear_que(struct nbd_device *nbd)
{
        blk_mq_quiesce_queue(nbd->disk->queue);
        blk_mq_tagset_busy_iter(&nbd->tag_set, nbd_clear_req, NULL);
        blk_mq_unquiesce_queue(nbd->disk->queue);
        dev_dbg(disk_to_dev(nbd->disk), "queue cleared\n");
}

static int find_fallback(struct nbd_device *nbd, int index)
{
        struct nbd_config *config = nbd->config;
        int new_index = -1;
        struct nbd_sock *nsock = config->socks[index];
        int fallback = nsock->fallback_index;

        if (test_bit(NBD_RT_DISCONNECTED, &config->runtime_flags))
                return new_index;

        if (config->num_connections <= 1) {
                dev_err_ratelimited(disk_to_dev(nbd->disk),
                                    "Dead connection, failed to find a fallback\n");
                return new_index;
        }

        if (fallback >= 0 && fallback < config->num_connections &&
            !config->socks[fallback]->dead)
                return fallback;

        if (nsock->fallback_index < 0 ||
            nsock->fallback_index >= config->num_connections ||
            config->socks[nsock->fallback_index]->dead) {
                int i;
                for (i = 0; i < config->num_connections; i++) {
                        if (i == index)
                                continue;
                        if (!config->socks[i]->dead) {
                                new_index = i;
                                break;
                        }
                }
                nsock->fallback_index = new_index;
                if (new_index < 0) {
                        dev_err_ratelimited(disk_to_dev(nbd->disk),
                                            "Dead connection, failed to find a fallback\n");
                        return new_index;
                }
        }
        new_index = nsock->fallback_index;
        return new_index;
}

static int wait_for_reconnect(struct nbd_device *nbd)
{
        struct nbd_config *config = nbd->config;
        if (!config->dead_conn_timeout)
                return 0;

        if (!wait_event_timeout(config->conn_wait,
                                test_bit(NBD_RT_DISCONNECTED,
                                         &config->runtime_flags) ||
                                atomic_read(&config->live_connections) > 0,
                                config->dead_conn_timeout))
                return 0;

        return !test_bit(NBD_RT_DISCONNECTED, &config->runtime_flags);
}

static blk_status_t nbd_handle_cmd(struct nbd_cmd *cmd, int index)
{
        struct request *req = blk_mq_rq_from_pdu(cmd);
        struct nbd_device *nbd = cmd->nbd;
        struct nbd_config *config;
        struct nbd_sock *nsock;
        blk_status_t ret;

        lockdep_assert_held(&cmd->lock);

        config = nbd_get_config_unlocked(nbd);
        if (!config) {
                dev_err_ratelimited(disk_to_dev(nbd->disk),
                                    "Socks array is empty\n");
                return BLK_STS_IOERR;
        }

        if (index >= config->num_connections) {
                dev_err_ratelimited(disk_to_dev(nbd->disk),
                                    "Attempted send on invalid socket\n");
                nbd_config_put(nbd);
                return BLK_STS_IOERR;
        }
        cmd->status = BLK_STS_OK;
again:
        nsock = config->socks[index];
        mutex_lock(&nsock->tx_lock);
        if (nsock->dead) {
                int old_index = index;
                index = find_fallback(nbd, index);
                mutex_unlock(&nsock->tx_lock);
                if (index < 0) {
                        if (wait_for_reconnect(nbd)) {
                                index = old_index;
                                goto again;
                        }
                        /* All the sockets should already be down at this point,
                         * we just want to make sure that DISCONNECTED is set so
                         * any requests that come in that were queue'ed waiting
                         * for the reconnect timer don't trigger the timer again
                         * and instead just error out.
                         */
                        sock_shutdown(nbd);
                        nbd_config_put(nbd);
                        return BLK_STS_IOERR;
                }
                goto again;
        }

        /* Handle the case that we have a pending request that was partially
         * transmitted that _has_ to be serviced first.  We need to call requeue
         * here so that it gets put _after_ the request that is already on the
         * dispatch list.
         */
        blk_mq_start_request(req);
        if (unlikely(nsock->pending && nsock->pending != req)) {
                nbd_requeue_cmd(cmd);
                ret = BLK_STS_OK;
                goto out;
        }
        ret = nbd_send_cmd(nbd, cmd, index);
out:
        mutex_unlock(&nsock->tx_lock);
        nbd_config_put(nbd);
        return ret;
}

static blk_status_t nbd_queue_rq(struct blk_mq_hw_ctx *hctx,
                        const struct blk_mq_queue_data *bd)
{
        struct nbd_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
        blk_status_t ret;

        /*
         * Since we look at the bio's to send the request over the network we
         * need to make sure the completion work doesn't mark this request done
         * before we are done doing our send.  This keeps us from dereferencing
         * freed data if we have particularly fast completions (ie we get the
         * completion before we exit sock_xmit on the last bvec) or in the case
         * that the server is misbehaving (or there was an error) before we're
         * done sending everything over the wire.
         */
        mutex_lock(&cmd->lock);
        clear_bit(NBD_CMD_REQUEUED, &cmd->flags);

        /* We can be called directly from the user space process, which means we
         * could possibly have signals pending so our sendmsg will fail.  In
         * this case we need to return that we are busy, otherwise error out as
         * appropriate.
         */
        ret = nbd_handle_cmd(cmd, hctx->queue_num);
        mutex_unlock(&cmd->lock);

        return ret;
}

static struct socket *nbd_get_socket(struct nbd_device *nbd, unsigned long fd,
                                     int *err)
{
        struct socket *sock;

        *err = 0;
        sock = sockfd_lookup(fd, err);
        if (!sock)
                return NULL;

        if (sock->ops->shutdown == sock_no_shutdown) {
                dev_err(disk_to_dev(nbd->disk), "Unsupported socket: shutdown callout must be supported.\n");
                *err = -EINVAL;
                sockfd_put(sock);
                return NULL;
        }

        return sock;
}

static int nbd_add_socket(struct nbd_device *nbd, unsigned long arg,
                          bool netlink)
{
        struct nbd_config *config = nbd->config;
        struct socket *sock;
        struct nbd_sock **socks;
        struct nbd_sock *nsock;
        unsigned int memflags;
        int err;

        /* Arg will be cast to int, check it to avoid overflow */
        if (arg > INT_MAX)
                return -EINVAL;
        sock = nbd_get_socket(nbd, arg, &err);
        if (!sock)
                return err;

        /*
         * We need to make sure we don't get any errant requests while we're
         * reallocating the ->socks array.
         */
        memflags = blk_mq_freeze_queue(nbd->disk->queue);

        if (!netlink && !nbd->task_setup &&
            !test_bit(NBD_RT_BOUND, &config->runtime_flags))
                nbd->task_setup = current;

        if (!netlink &&
            (nbd->task_setup != current ||
             test_bit(NBD_RT_BOUND, &config->runtime_flags))) {
                dev_err(disk_to_dev(nbd->disk),
                        "Device being setup by another task");
                err = -EBUSY;
                goto put_socket;
        }

        nsock = kzalloc(sizeof(*nsock), GFP_KERNEL);
        if (!nsock) {
                err = -ENOMEM;
                goto put_socket;
        }

        socks = krealloc(config->socks, (config->num_connections + 1) *
                         sizeof(struct nbd_sock *), GFP_KERNEL);
        if (!socks) {
                kfree(nsock);
                err = -ENOMEM;
                goto put_socket;
        }

        config->socks = socks;

        nsock->fallback_index = -1;
        nsock->dead = false;
        mutex_init(&nsock->tx_lock);
        nsock->sock = sock;
        nsock->pending = NULL;
        nsock->sent = 0;
        nsock->cookie = 0;
        INIT_WORK(&nsock->work, nbd_pending_cmd_work);
        socks[config->num_connections++] = nsock;
        atomic_inc(&config->live_connections);
        blk_mq_unfreeze_queue(nbd->disk->queue, memflags);

        return 0;

put_socket:
        blk_mq_unfreeze_queue(nbd->disk->queue, memflags);
        sockfd_put(sock);
        return err;
}

static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg)
{
        struct nbd_config *config = nbd->config;
        struct socket *sock, *old;
        struct recv_thread_args *args;
        int i;
        int err;

        sock = nbd_get_socket(nbd, arg, &err);
        if (!sock)
                return err;

        args = kzalloc(sizeof(*args), GFP_KERNEL);
        if (!args) {
                sockfd_put(sock);
                return -ENOMEM;
        }

        for (i = 0; i < config->num_connections; i++) {
                struct nbd_sock *nsock = config->socks[i];

                if (!nsock->dead)
                        continue;

                mutex_lock(&nsock->tx_lock);
                if (!nsock->dead) {
                        mutex_unlock(&nsock->tx_lock);
                        continue;
                }
                sk_set_memalloc(sock->sk);
                if (nbd->tag_set.timeout)
                        sock->sk->sk_sndtimeo = nbd->tag_set.timeout;
                atomic_inc(&config->recv_threads);
                refcount_inc(&nbd->config_refs);
                old = nsock->sock;
                nsock->fallback_index = -1;
                nsock->sock = sock;
                nsock->dead = false;
                INIT_WORK(&args->work, recv_work);
                args->index = i;
                args->nbd = nbd;
                args->nsock = nsock;
                nsock->cookie++;
                mutex_unlock(&nsock->tx_lock);
                sockfd_put(old);

                clear_bit(NBD_RT_DISCONNECTED, &config->runtime_flags);

                /* We take the tx_mutex in an error path in the recv_work, so we
                 * need to queue_work outside of the tx_mutex.
                 */
                queue_work(nbd->recv_workq, &args->work);

                atomic_inc(&config->live_connections);
                wake_up(&config->conn_wait);
                return 0;
        }
        sockfd_put(sock);
        kfree(args);
        return -ENOSPC;
}

static void nbd_bdev_reset(struct nbd_device *nbd)
{
        if (disk_openers(nbd->disk) > 1)
                return;
        set_capacity(nbd->disk, 0);
}

static void nbd_parse_flags(struct nbd_device *nbd)
{
        if (nbd->config->flags & NBD_FLAG_READ_ONLY)
                set_disk_ro(nbd->disk, true);
        else
                set_disk_ro(nbd->disk, false);
}

static void send_disconnects(struct nbd_device *nbd)
{
        struct nbd_config *config = nbd->config;
        struct nbd_request request = {
                .magic = htonl(NBD_REQUEST_MAGIC),
                .type = htonl(NBD_CMD_DISC),
        };
        struct kvec iov = {.iov_base = &request, .iov_len = sizeof(request)};
        struct iov_iter from;
        int i, ret;

        for (i = 0; i < config->num_connections; i++) {
                struct nbd_sock *nsock = config->socks[i];

                iov_iter_kvec(&from, ITER_SOURCE, &iov, 1, sizeof(request));
                mutex_lock(&nsock->tx_lock);
                ret = sock_xmit(nbd, i, 1, &from, 0, NULL);
                if (ret < 0)
                        dev_err(disk_to_dev(nbd->disk),
                                "Send disconnect failed %d\n", ret);
                mutex_unlock(&nsock->tx_lock);
        }
}

static int nbd_disconnect(struct nbd_device *nbd)
{
        struct nbd_config *config = nbd->config;

        dev_info(disk_to_dev(nbd->disk), "NBD_DISCONNECT\n");
        set_bit(NBD_RT_DISCONNECT_REQUESTED, &config->runtime_flags);
        set_bit(NBD_DISCONNECT_REQUESTED, &nbd->flags);
        send_disconnects(nbd);
        return 0;
}

static void nbd_clear_sock(struct nbd_device *nbd)
{
        sock_shutdown(nbd);
        nbd_clear_que(nbd);
        nbd->task_setup = NULL;
}

static void nbd_config_put(struct nbd_device *nbd)
{
        if (refcount_dec_and_mutex_lock(&nbd->config_refs,
                                        &nbd->config_lock)) {
                struct nbd_config *config = nbd->config;
                nbd_dev_dbg_close(nbd);
                invalidate_disk(nbd->disk);
                if (nbd->config->bytesize)
                        kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE);
                if (test_and_clear_bit(NBD_RT_HAS_PID_FILE,
                                       &config->runtime_flags))
                        device_remove_file(disk_to_dev(nbd->disk), &pid_attr);
                nbd->pid = 0;
                if (test_and_clear_bit(NBD_RT_HAS_BACKEND_FILE,
                                       &config->runtime_flags)) {
                        device_remove_file(disk_to_dev(nbd->disk), &backend_attr);
                        kfree(nbd->backend);
                        nbd->backend = NULL;
                }
                nbd_clear_sock(nbd);
                if (config->num_connections) {
                        int i;
                        for (i = 0; i < config->num_connections; i++) {
                                sockfd_put(config->socks[i]->sock);
                                kfree(config->socks[i]);
                        }
                        kfree(config->socks);
                }
                kfree(nbd->config);
                nbd->config = NULL;

                nbd->tag_set.timeout = 0;

                mutex_unlock(&nbd->config_lock);
                nbd_put(nbd);
                module_put(THIS_MODULE);
        }
}

static int nbd_start_device(struct nbd_device *nbd)
{
        struct nbd_config *config = nbd->config;
        int num_connections = config->num_connections;
        int error = 0, i;

        if (nbd->pid)
                return -EBUSY;
        if (!config->socks)
                return -EINVAL;
        if (num_connections > 1 &&
            !(config->flags & NBD_FLAG_CAN_MULTI_CONN)) {
                dev_err(disk_to_dev(nbd->disk), "server does not support multiple connections per device.\n");
                return -EINVAL;
        }

        blk_mq_update_nr_hw_queues(&nbd->tag_set, config->num_connections);
        nbd->pid = task_pid_nr(current);

        nbd_parse_flags(nbd);

        error = device_create_file(disk_to_dev(nbd->disk), &pid_attr);
        if (error) {
                dev_err(disk_to_dev(nbd->disk), "device_create_file failed for pid!\n");
                return error;
        }
        set_bit(NBD_RT_HAS_PID_FILE, &config->runtime_flags);

        nbd_dev_dbg_init(nbd);
        for (i = 0; i < num_connections; i++) {
                struct recv_thread_args *args;

                args = kzalloc(sizeof(*args), GFP_KERNEL);
                if (!args) {
                        sock_shutdown(nbd);
                        /*
                         * If num_connections is m (2 < m),
                         * and NO.1 ~ NO.n(1 < n < m) kzallocs are successful.
                         * But NO.(n + 1) failed. We still have n recv threads.
                         * So, add flush_workqueue here to prevent recv threads
                         * dropping the last config_refs and trying to destroy
                         * the workqueue from inside the workqueue.
                         */
                        if (i)
                                flush_workqueue(nbd->recv_workq);
                        return -ENOMEM;
                }
                sk_set_memalloc(config->socks[i]->sock->sk);
                if (nbd->tag_set.timeout)
                        config->socks[i]->sock->sk->sk_sndtimeo =
                                nbd->tag_set.timeout;
                atomic_inc(&config->recv_threads);
                refcount_inc(&nbd->config_refs);
                INIT_WORK(&args->work, recv_work);
                args->nbd = nbd;
                args->nsock = config->socks[i];
                args->index = i;
                queue_work(nbd->recv_workq, &args->work);
        }
        return nbd_set_size(nbd, config->bytesize, nbd_blksize(config));
}

static int nbd_start_device_ioctl(struct nbd_device *nbd)
{
        struct nbd_config *config = nbd->config;
        int ret;

        ret = nbd_start_device(nbd);
        if (ret)
                return ret;

        if (max_part)
                set_bit(GD_NEED_PART_SCAN, &nbd->disk->state);
        mutex_unlock(&nbd->config_lock);
        ret = wait_event_interruptible(config->recv_wq,
                                         atomic_read(&config->recv_threads) == 0);
        if (ret) {
                sock_shutdown(nbd);
                nbd_clear_que(nbd);
        }

        flush_workqueue(nbd->recv_workq);
        mutex_lock(&nbd->config_lock);
        nbd_bdev_reset(nbd);
        /* user requested, ignore socket errors */
        if (test_bit(NBD_RT_DISCONNECT_REQUESTED, &config->runtime_flags))
                ret = 0;
        if (test_bit(NBD_RT_TIMEDOUT, &config->runtime_flags))
                ret = -ETIMEDOUT;
        return ret;
}

static void nbd_clear_sock_ioctl(struct nbd_device *nbd)
{
        nbd_clear_sock(nbd);
        disk_force_media_change(nbd->disk);
        nbd_bdev_reset(nbd);
        if (test_and_clear_bit(NBD_RT_HAS_CONFIG_REF,
                               &nbd->config->runtime_flags))
                nbd_config_put(nbd);
}

static void nbd_set_cmd_timeout(struct nbd_device *nbd, u64 timeout)
{
        nbd->tag_set.timeout = timeout * HZ;
        if (timeout)
                blk_queue_rq_timeout(nbd->disk->queue, timeout * HZ);
        else
                blk_queue_rq_timeout(nbd->disk->queue, 30 * HZ);
}

/* Must be called with config_lock held */
static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,
                       unsigned int cmd, unsigned long arg)
{
        struct nbd_config *config = nbd->config;
        loff_t bytesize;

        switch (cmd) {
        case NBD_DISCONNECT:
                return nbd_disconnect(nbd);
        case NBD_CLEAR_SOCK:
                nbd_clear_sock_ioctl(nbd);
                return 0;
        case NBD_SET_SOCK:
                return nbd_add_socket(nbd, arg, false);
        case NBD_SET_BLKSIZE:
                return nbd_set_size(nbd, config->bytesize, arg);
        case NBD_SET_SIZE:
                return nbd_set_size(nbd, arg, nbd_blksize(config));
        case NBD_SET_SIZE_BLOCKS:
                if (check_shl_overflow(arg, config->blksize_bits, &bytesize))
                        return -EINVAL;
                return nbd_set_size(nbd, bytesize, nbd_blksize(config));
        case NBD_SET_TIMEOUT:
                nbd_set_cmd_timeout(nbd, arg);
                return 0;

        case NBD_SET_FLAGS:
                config->flags = arg;
                return 0;
        case NBD_DO_IT:
                return nbd_start_device_ioctl(nbd);
        case NBD_CLEAR_QUE:
                /*
                 * This is for compatibility only.  The queue is always cleared
                 * by NBD_DO_IT or NBD_CLEAR_SOCK.
                 */
                return 0;
        case NBD_PRINT_DEBUG:
                /*
                 * For compatibility only, we no longer keep a list of
                 * outstanding requests.
                 */
                return 0;
        }
        return -ENOTTY;
}

static int nbd_ioctl(struct block_device *bdev, blk_mode_t mode,
                     unsigned int cmd, unsigned long arg)
{
        struct nbd_device *nbd = bdev->bd_disk->private_data;
        struct nbd_config *config = nbd->config;
        int error = -EINVAL;

        if (!capable(CAP_SYS_ADMIN))
                return -EPERM;

        /* The block layer will pass back some non-nbd ioctls in case we have
         * special handling for them, but we don't so just return an error.
         */
        if (_IOC_TYPE(cmd) != 0xab)
                return -EINVAL;

        mutex_lock(&nbd->config_lock);

        /* Don't allow ioctl operations on a nbd device that was created with
         * netlink, unless it's DISCONNECT or CLEAR_SOCK, which are fine.
         */
        if (!test_bit(NBD_RT_BOUND, &config->runtime_flags) ||
            (cmd == NBD_DISCONNECT || cmd == NBD_CLEAR_SOCK))
                error = __nbd_ioctl(bdev, nbd, cmd, arg);
        else
                dev_err(nbd_to_dev(nbd), "Cannot use ioctl interface on a netlink controlled device.\n");
        mutex_unlock(&nbd->config_lock);
        return error;
}

static int nbd_alloc_and_init_config(struct nbd_device *nbd)
{
        struct nbd_config *config;

        if (WARN_ON(nbd->config))
                return -EINVAL;

        if (!try_module_get(THIS_MODULE))
                return -ENODEV;

        config = kzalloc(sizeof(struct nbd_config), GFP_NOFS);
        if (!config) {
                module_put(THIS_MODULE);
                return -ENOMEM;
        }

        atomic_set(&config->recv_threads, 0);
        init_waitqueue_head(&config->recv_wq);
        init_waitqueue_head(&config->conn_wait);
        config->blksize_bits = NBD_DEF_BLKSIZE_BITS;
        atomic_set(&config->live_connections, 0);

        nbd->config = config;
        /*
         * Order refcount_set(&nbd->config_refs, 1) and nbd->config assignment,
         * its pair is the barrier in nbd_get_config_unlocked().
         * So nbd_get_config_unlocked() won't see nbd->config as null after
         * refcount_inc_not_zero() succeed.
         */
        smp_mb__before_atomic();
        refcount_set(&nbd->config_refs, 1);

        return 0;
}

static int nbd_open(struct gendisk *disk, blk_mode_t mode)
{
        struct nbd_device *nbd;
        struct nbd_config *config;
        int ret = 0;

        mutex_lock(&nbd_index_mutex);
        nbd = disk->private_data;
        if (!nbd) {
                ret = -ENXIO;
                goto out;
        }
        if (!refcount_inc_not_zero(&nbd->refs)) {
                ret = -ENXIO;
                goto out;
        }

        config = nbd_get_config_unlocked(nbd);
        if (!config) {
                mutex_lock(&nbd->config_lock);
                if (refcount_inc_not_zero(&nbd->config_refs)) {
                        mutex_unlock(&nbd->config_lock);
                        goto out;
                }
                ret = nbd_alloc_and_init_config(nbd);
                if (ret) {
                        mutex_unlock(&nbd->config_lock);
                        goto out;
                }

                refcount_inc(&nbd->refs);
                mutex_unlock(&nbd->config_lock);
                if (max_part)
                        set_bit(GD_NEED_PART_SCAN, &disk->state);
        } else if (nbd_disconnected(config)) {
                if (max_part)
                        set_bit(GD_NEED_PART_SCAN, &disk->state);
        }
out:
        mutex_unlock(&nbd_index_mutex);
        return ret;
}

static void nbd_release(struct gendisk *disk)
{
        struct nbd_device *nbd = disk->private_data;

        if (test_bit(NBD_RT_DISCONNECT_ON_CLOSE, &nbd->config->runtime_flags) &&
                        disk_openers(disk) == 0)
                nbd_disconnect_and_put(nbd);

        nbd_config_put(nbd);
        nbd_put(nbd);
}

static void nbd_free_disk(struct gendisk *disk)
{
        struct nbd_device *nbd = disk->private_data;

        kfree(nbd);
}

static const struct block_device_operations nbd_fops =
{
        .owner =        THIS_MODULE,
        .open =                nbd_open,
        .release =        nbd_release,
        .ioctl =        nbd_ioctl,
        .compat_ioctl =        nbd_ioctl,
        .free_disk =        nbd_free_disk,
};

#if IS_ENABLED(CONFIG_DEBUG_FS)

static int nbd_dbg_tasks_show(struct seq_file *s, void *unused)
{
        struct nbd_device *nbd = s->private;

        if (nbd->pid)
                seq_printf(s, "recv: %d\n", nbd->pid);

        return 0;
}

DEFINE_SHOW_ATTRIBUTE(nbd_dbg_tasks);

static int nbd_dbg_flags_show(struct seq_file *s, void *unused)
{
        struct nbd_device *nbd = s->private;
        u32 flags = nbd->config->flags;

        seq_printf(s, "Hex: 0x%08x\n\n", flags);

        seq_puts(s, "Known flags:\n");

        if (flags & NBD_FLAG_HAS_FLAGS)
                seq_puts(s, "NBD_FLAG_HAS_FLAGS\n");
        if (flags & NBD_FLAG_READ_ONLY)
                seq_puts(s, "NBD_FLAG_READ_ONLY\n");
        if (flags & NBD_FLAG_SEND_FLUSH)
                seq_puts(s, "NBD_FLAG_SEND_FLUSH\n");
        if (flags & NBD_FLAG_SEND_FUA)
                seq_puts(s, "NBD_FLAG_SEND_FUA\n");
        if (flags & NBD_FLAG_SEND_TRIM)
                seq_puts(s, "NBD_FLAG_SEND_TRIM\n");
        if (flags & NBD_FLAG_SEND_WRITE_ZEROES)
                seq_puts(s, "NBD_FLAG_SEND_WRITE_ZEROES\n");
        if (flags & NBD_FLAG_ROTATIONAL)
                seq_puts(s, "NBD_FLAG_ROTATIONAL\n");

        return 0;
}

DEFINE_SHOW_ATTRIBUTE(nbd_dbg_flags);

static int nbd_dev_dbg_init(struct nbd_device *nbd)
{
        struct dentry *dir;
        struct nbd_config *config = nbd->config;

        if (!nbd_dbg_dir)
                return -EIO;

        dir = debugfs_create_dir(nbd_name(nbd), nbd_dbg_dir);
        if (IS_ERR(dir)) {
                dev_err(nbd_to_dev(nbd), "Failed to create debugfs dir for '%s'\n",
                        nbd_name(nbd));
                return -EIO;
        }
        config->dbg_dir = dir;

        debugfs_create_file("tasks", 0444, dir, nbd, &nbd_dbg_tasks_fops);
        debugfs_create_u64("size_bytes", 0444, dir, &config->bytesize);
        debugfs_create_u32("timeout", 0444, dir, &nbd->tag_set.timeout);
        debugfs_create_u32("blocksize_bits", 0444, dir, &config->blksize_bits);
        debugfs_create_file("flags", 0444, dir, nbd, &nbd_dbg_flags_fops);

        return 0;
}

static void nbd_dev_dbg_close(struct nbd_device *nbd)
{
        debugfs_remove_recursive(nbd->config->dbg_dir);
}

static int nbd_dbg_init(void)
{
        struct dentry *dbg_dir;

        dbg_dir = debugfs_create_dir("nbd", NULL);
        if (IS_ERR(dbg_dir))
                return -EIO;

        nbd_dbg_dir = dbg_dir;

        return 0;
}

static void nbd_dbg_close(void)
{
        debugfs_remove_recursive(nbd_dbg_dir);
}

#else  /* IS_ENABLED(CONFIG_DEBUG_FS) */

static int nbd_dev_dbg_init(struct nbd_device *nbd)
{
        return 0;
}

static void nbd_dev_dbg_close(struct nbd_device *nbd)
{
}

static int nbd_dbg_init(void)
{
        return 0;
}

static void nbd_dbg_close(void)
{
}

#endif

static int nbd_init_request(struct blk_mq_tag_set *set, struct request *rq,
                            unsigned int hctx_idx, unsigned int numa_node)
{
        struct nbd_cmd *cmd = blk_mq_rq_to_pdu(rq);
        cmd->nbd = set->driver_data;
        cmd->flags = 0;
        mutex_init(&cmd->lock);
        return 0;
}

static const struct blk_mq_ops nbd_mq_ops = {
        .queue_rq        = nbd_queue_rq,
        .complete        = nbd_complete_rq,
        .init_request        = nbd_init_request,
        .timeout        = nbd_xmit_timeout,
};

static struct nbd_device *nbd_dev_add(int index, unsigned int refs)
{
        struct queue_limits lim = {
                .max_hw_sectors                = 65536,
                .io_opt                        = 256 << SECTOR_SHIFT,
                .max_segments                = USHRT_MAX,
                .max_segment_size        = UINT_MAX,
        };
        struct nbd_device *nbd;
        struct gendisk *disk;
        int err = -ENOMEM;

        nbd = kzalloc(sizeof(struct nbd_device), GFP_KERNEL);
        if (!nbd)
                goto out;

        nbd->tag_set.ops = &nbd_mq_ops;
        nbd->tag_set.nr_hw_queues = 1;
        nbd->tag_set.queue_depth = 128;
        nbd->tag_set.numa_node = NUMA_NO_NODE;
        nbd->tag_set.cmd_size = sizeof(struct nbd_cmd);
        nbd->tag_set.flags = BLK_MQ_F_BLOCKING;
        nbd->tag_set.driver_data = nbd;
        INIT_WORK(&nbd->remove_work, nbd_dev_remove_work);
        nbd->backend = NULL;

        err = blk_mq_alloc_tag_set(&nbd->tag_set);
        if (err)
                goto out_free_nbd;

        mutex_lock(&nbd_index_mutex);
        if (index >= 0) {
                err = idr_alloc(&nbd_index_idr, nbd, index, index + 1,
                                GFP_KERNEL);
                if (err == -ENOSPC)
                        err = -EEXIST;
        } else {
                err = idr_alloc(&nbd_index_idr, nbd, 0,
                                (MINORMASK >> part_shift) + 1, GFP_KERNEL);
                if (err >= 0)
                        index = err;
        }
        nbd->index = index;
        mutex_unlock(&nbd_index_mutex);
        if (err < 0)
                goto out_free_tags;

        disk = blk_mq_alloc_disk(&nbd->tag_set, &lim, NULL);
        if (IS_ERR(disk)) {
                err = PTR_ERR(disk);
                goto out_free_idr;
        }
        nbd->disk = disk;

        nbd->recv_workq = alloc_workqueue("nbd%d-recv",
                                          WQ_MEM_RECLAIM | WQ_HIGHPRI |
                                          WQ_UNBOUND, 0, nbd->index);
        if (!nbd->recv_workq) {
                dev_err(disk_to_dev(nbd->disk), "Could not allocate knbd recv work queue.\n");
                err = -ENOMEM;
                goto out_err_disk;
        }

        mutex_init(&nbd->config_lock);
        refcount_set(&nbd->config_refs, 0);
        /*
         * Start out with a zero references to keep other threads from using
         * this device until it is fully initialized.
         */
        refcount_set(&nbd->refs, 0);
        INIT_LIST_HEAD(&nbd->list);
        disk->major = NBD_MAJOR;
        disk->first_minor = index << part_shift;
        disk->minors = 1 << part_shift;
        disk->fops = &nbd_fops;
        disk->private_data = nbd;
        sprintf(disk->disk_name, "nbd%d", index);
        err = add_disk(disk);
        if (err)
                goto out_free_work;

        /*
         * Now publish the device.
         */
        refcount_set(&nbd->refs, refs);
        nbd_total_devices++;
        return nbd;

out_free_work:
        destroy_workqueue(nbd->recv_workq);
out_err_disk:
        put_disk(disk);
out_free_idr:
        mutex_lock(&nbd_index_mutex);
        idr_remove(&nbd_index_idr, index);
        mutex_unlock(&nbd_index_mutex);
out_free_tags:
        blk_mq_free_tag_set(&nbd->tag_set);
out_free_nbd:
        kfree(nbd);
out:
        return ERR_PTR(err);
}

static struct nbd_device *nbd_find_get_unused(void)
{
        struct nbd_device *nbd;
        int id;

        lockdep_assert_held(&nbd_index_mutex);

        idr_for_each_entry(&nbd_index_idr, nbd, id) {
                if (refcount_read(&nbd->config_refs) ||
                    test_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags))
                        continue;
                if (refcount_inc_not_zero(&nbd->refs))
                        return nbd;
        }

        return NULL;
}

/* Netlink interface. */
static const struct nla_policy nbd_attr_policy[NBD_ATTR_MAX + 1] = {
        [NBD_ATTR_INDEX]                =        { .type = NLA_U32 },
        [NBD_ATTR_SIZE_BYTES]                =        { .type = NLA_U64 },
        [NBD_ATTR_BLOCK_SIZE_BYTES]        =        { .type = NLA_U64 },
        [NBD_ATTR_TIMEOUT]                =        { .type = NLA_U64 },
        [NBD_ATTR_SERVER_FLAGS]                =        { .type = NLA_U64 },
        [NBD_ATTR_CLIENT_FLAGS]                =        { .type = NLA_U64 },
        [NBD_ATTR_SOCKETS]                =        { .type = NLA_NESTED},
        [NBD_ATTR_DEAD_CONN_TIMEOUT]        =        { .type = NLA_U64 },
        [NBD_ATTR_DEVICE_LIST]                =        { .type = NLA_NESTED},
        [NBD_ATTR_BACKEND_IDENTIFIER]        =        { .type = NLA_STRING},
};

static const struct nla_policy nbd_sock_policy[NBD_SOCK_MAX + 1] = {
        [NBD_SOCK_FD]                        =        { .type = NLA_U32 },
};

/* We don't use this right now since we don't parse the incoming list, but we
 * still want it here so userspace knows what to expect.
 */
static const struct nla_policy __attribute__((unused))
nbd_device_policy[NBD_DEVICE_ATTR_MAX + 1] = {
        [NBD_DEVICE_INDEX]                =        { .type = NLA_U32 },
        [NBD_DEVICE_CONNECTED]                =        { .type = NLA_U8 },
};

static int nbd_genl_size_set(struct genl_info *info, struct nbd_device *nbd)
{
        struct nbd_config *config = nbd->config;
        u64 bsize = nbd_blksize(config);
        u64 bytes = config->bytesize;

        if (info->attrs[NBD_ATTR_SIZE_BYTES])
                bytes = nla_get_u64(info->attrs[NBD_ATTR_SIZE_BYTES]);

        if (info->attrs[NBD_ATTR_BLOCK_SIZE_BYTES])
                bsize = nla_get_u64(info->attrs[NBD_ATTR_BLOCK_SIZE_BYTES]);

        if (bytes != config->bytesize || bsize != nbd_blksize(config))
                return nbd_set_size(nbd, bytes, bsize);
        return 0;
}

static int nbd_genl_connect(struct sk_buff *skb, struct genl_info *info)
{
        struct nbd_device *nbd;
        struct nbd_config *config;
        int index = -1;
        int ret;
        bool put_dev = false;

        if (!netlink_capable(skb, CAP_SYS_ADMIN))
                return -EPERM;

        if (info->attrs[NBD_ATTR_INDEX]) {
                index = nla_get_u32(info->attrs[NBD_ATTR_INDEX]);

                /*
                 * Too big first_minor can cause duplicate creation of
                 * sysfs files/links, since index << part_shift might overflow, or
                 * MKDEV() expect that the max bits of first_minor is 20.
                 */
                if (index < 0 || index > MINORMASK >> part_shift) {
                        pr_err("illegal input index %d\n", index);
                        return -EINVAL;
                }
        }
        if (GENL_REQ_ATTR_CHECK(info, NBD_ATTR_SOCKETS)) {
                pr_err("must specify at least one socket\n");
                return -EINVAL;
        }
        if (GENL_REQ_ATTR_CHECK(info, NBD_ATTR_SIZE_BYTES)) {
                pr_err("must specify a size in bytes for the device\n");
                return -EINVAL;
        }
again:
        mutex_lock(&nbd_index_mutex);
        if (index == -1) {
                nbd = nbd_find_get_unused();
        } else {
                nbd = idr_find(&nbd_index_idr, index);
                if (nbd) {
                        if ((test_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags) &&
                             test_bit(NBD_DISCONNECT_REQUESTED, &nbd->flags)) ||
                            !refcount_inc_not_zero(&nbd->refs)) {
                                mutex_unlock(&nbd_index_mutex);
                                pr_err("device at index %d is going down\n",
                                        index);
                                return -EINVAL;
                        }
                }
        }
        mutex_unlock(&nbd_index_mutex);

        if (!nbd) {
                nbd = nbd_dev_add(index, 2);
                if (IS_ERR(nbd)) {
                        pr_err("failed to add new device\n");
                        return PTR_ERR(nbd);
                }
        }

        mutex_lock(&nbd->config_lock);
        if (refcount_read(&nbd->config_refs)) {
                mutex_unlock(&nbd->config_lock);
                nbd_put(nbd);
                if (index == -1)
                        goto again;
                pr_err("nbd%d already in use\n", index);
                return -EBUSY;
        }

        ret = nbd_alloc_and_init_config(nbd);
        if (ret) {
                mutex_unlock(&nbd->config_lock);
                nbd_put(nbd);
                pr_err("couldn't allocate config\n");
                return ret;
        }

        config = nbd->config;
        set_bit(NBD_RT_BOUND, &config->runtime_flags);
        ret = nbd_genl_size_set(info, nbd);
        if (ret)
                goto out;

        if (info->attrs[NBD_ATTR_TIMEOUT])
                nbd_set_cmd_timeout(nbd,
                                    nla_get_u64(info->attrs[NBD_ATTR_TIMEOUT]));
        if (info->attrs[NBD_ATTR_DEAD_CONN_TIMEOUT]) {
                config->dead_conn_timeout =
                        nla_get_u64(info->attrs[NBD_ATTR_DEAD_CONN_TIMEOUT]);
                config->dead_conn_timeout *= HZ;
        }
        if (info->attrs[NBD_ATTR_SERVER_FLAGS])
                config->flags =
                        nla_get_u64(info->attrs[NBD_ATTR_SERVER_FLAGS]);
        if (info->attrs[NBD_ATTR_CLIENT_FLAGS]) {
                u64 flags = nla_get_u64(info->attrs[NBD_ATTR_CLIENT_FLAGS]);
                if (flags & NBD_CFLAG_DESTROY_ON_DISCONNECT) {
                        /*
                         * We have 1 ref to keep the device around, and then 1
                         * ref for our current operation here, which will be
                         * inherited by the config.  If we already have
                         * DESTROY_ON_DISCONNECT set then we know we don't have
                         * that extra ref already held so we don't need the
                         * put_dev.
                         */
                        if (!test_and_set_bit(NBD_DESTROY_ON_DISCONNECT,
                                              &nbd->flags))
                                put_dev = true;
                } else {
                        if (test_and_clear_bit(NBD_DESTROY_ON_DISCONNECT,
                                               &nbd->flags))
                                refcount_inc(&nbd->refs);
                }
                if (flags & NBD_CFLAG_DISCONNECT_ON_CLOSE) {
                        set_bit(NBD_RT_DISCONNECT_ON_CLOSE,
                                &config->runtime_flags);
                }
        }

        if (info->attrs[NBD_ATTR_SOCKETS]) {
                struct nlattr *attr;
                int rem, fd;

                nla_for_each_nested(attr, info->attrs[NBD_ATTR_SOCKETS],
                                    rem) {
                        struct nlattr *socks[NBD_SOCK_MAX+1];

                        if (nla_type(attr) != NBD_SOCK_ITEM) {
                                pr_err("socks must be embedded in a SOCK_ITEM attr\n");
                                ret = -EINVAL;
                                goto out;
                        }
                        ret = nla_parse_nested_deprecated(socks, NBD_SOCK_MAX,
                                                          attr,
                                                          nbd_sock_policy,
                                                          info->extack);
                        if (ret != 0) {
                                pr_err("error processing sock list\n");
                                ret = -EINVAL;
                                goto out;
                        }
                        if (!socks[NBD_SOCK_FD])
                                continue;
                        fd = (int)nla_get_u32(socks[NBD_SOCK_FD]);
                        ret = nbd_add_socket(nbd, fd, true);
                        if (ret)
                                goto out;
                }
        }
        ret = nbd_start_device(nbd);
        if (ret)
                goto out;
        if (info->attrs[NBD_ATTR_BACKEND_IDENTIFIER]) {
                nbd->backend = nla_strdup(info->attrs[NBD_ATTR_BACKEND_IDENTIFIER],
                                          GFP_KERNEL);
                if (!nbd->backend) {
                        ret = -ENOMEM;
                        goto out;
                }
        }
        ret = device_create_file(disk_to_dev(nbd->disk), &backend_attr);
        if (ret) {
                dev_err(disk_to_dev(nbd->disk),
                        "device_create_file failed for backend!\n");
                goto out;
        }
        set_bit(NBD_RT_HAS_BACKEND_FILE, &config->runtime_flags);
out:
        mutex_unlock(&nbd->config_lock);
        if (!ret) {
                set_bit(NBD_RT_HAS_CONFIG_REF, &config->runtime_flags);
                refcount_inc(&nbd->config_refs);
                nbd_connect_reply(info, nbd->index);
        }
        nbd_config_put(nbd);
        if (put_dev)
                nbd_put(nbd);
        return ret;
}

static void nbd_disconnect_and_put(struct nbd_device *nbd)
{
        mutex_lock(&nbd->config_lock);
        nbd_disconnect(nbd);
        sock_shutdown(nbd);
        wake_up(&nbd->config->conn_wait);
        /*
         * Make sure recv thread has finished, we can safely call nbd_clear_que()
         * to cancel the inflight I/Os.
         */
        flush_workqueue(nbd->recv_workq);
        nbd_clear_que(nbd);
        nbd->task_setup = NULL;
        clear_bit(NBD_RT_BOUND, &nbd->config->runtime_flags);
        mutex_unlock(&nbd->config_lock);

        if (test_and_clear_bit(NBD_RT_HAS_CONFIG_REF,
                               &nbd->config->runtime_flags))
                nbd_config_put(nbd);
}

static int nbd_genl_disconnect(struct sk_buff *skb, struct genl_info *info)
{
        struct nbd_device *nbd;
        int index;

        if (!netlink_capable(skb, CAP_SYS_ADMIN))
                return -EPERM;

        if (GENL_REQ_ATTR_CHECK(info, NBD_ATTR_INDEX)) {
                pr_err("must specify an index to disconnect\n");
                return -EINVAL;
        }
        index = nla_get_u32(info->attrs[NBD_ATTR_INDEX]);
        mutex_lock(&nbd_index_mutex);
        nbd = idr_find(&nbd_index_idr, index);
        if (!nbd) {
                mutex_unlock(&nbd_index_mutex);
                pr_err("couldn't find device at index %d\n", index);
                return -EINVAL;
        }
        if (!refcount_inc_not_zero(&nbd->refs)) {
                mutex_unlock(&nbd_index_mutex);
                pr_err("device at index %d is going down\n", index);
                return -EINVAL;
        }
        mutex_unlock(&nbd_index_mutex);
        if (!refcount_inc_not_zero(&nbd->config_refs))
                goto put_nbd;
        nbd_disconnect_and_put(nbd);
        nbd_config_put(nbd);
put_nbd:
        nbd_put(nbd);
        return 0;
}

static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info)
{
        struct nbd_device *nbd = NULL;
        struct nbd_config *config;
        int index;
        int ret = 0;
        bool put_dev = false;

        if (!netlink_capable(skb, CAP_SYS_ADMIN))
                return -EPERM;

        if (GENL_REQ_ATTR_CHECK(info, NBD_ATTR_INDEX)) {
                pr_err("must specify a device to reconfigure\n");
                return -EINVAL;
        }
        index = nla_get_u32(info->attrs[NBD_ATTR_INDEX]);
        mutex_lock(&nbd_index_mutex);
        nbd = idr_find(&nbd_index_idr, index);
        if (!nbd) {
                mutex_unlock(&nbd_index_mutex);
                pr_err("couldn't find a device at index %d\n", index);
                return -EINVAL;
        }
        if (nbd->backend) {
                if (info->attrs[NBD_ATTR_BACKEND_IDENTIFIER]) {
                        if (nla_strcmp(info->attrs[NBD_ATTR_BACKEND_IDENTIFIER],
                                       nbd->backend)) {
                                mutex_unlock(&nbd_index_mutex);
                                dev_err(nbd_to_dev(nbd),
                                        "backend image doesn't match with %s\n",
                                        nbd->backend);
                                return -EINVAL;
                        }
                } else {
                        mutex_unlock(&nbd_index_mutex);
                        dev_err(nbd_to_dev(nbd), "must specify backend\n");
                        return -EINVAL;
                }
        }
        if (!refcount_inc_not_zero(&nbd->refs)) {
                mutex_unlock(&nbd_index_mutex);
                pr_err("device at index %d is going down\n", index);
                return -EINVAL;
        }
        mutex_unlock(&nbd_index_mutex);

        config = nbd_get_config_unlocked(nbd);
        if (!config) {
                dev_err(nbd_to_dev(nbd),
                        "not configured, cannot reconfigure\n");
                nbd_put(nbd);
                return -EINVAL;
        }

        mutex_lock(&nbd->config_lock);
        if (!test_bit(NBD_RT_BOUND, &config->runtime_flags) ||
            !nbd->pid) {
                dev_err(nbd_to_dev(nbd),
                        "not configured, cannot reconfigure\n");
                ret = -EINVAL;
                goto out;
        }

        ret = nbd_genl_size_set(info, nbd);
        if (ret)
                goto out;

        if (info->attrs[NBD_ATTR_TIMEOUT])
                nbd_set_cmd_timeout(nbd,
                                    nla_get_u64(info->attrs[NBD_ATTR_TIMEOUT]));
        if (info->attrs[NBD_ATTR_DEAD_CONN_TIMEOUT]) {
                config->dead_conn_timeout =
                        nla_get_u64(info->attrs[NBD_ATTR_DEAD_CONN_TIMEOUT]);
                config->dead_conn_timeout *= HZ;
        }
        if (info->attrs[NBD_ATTR_CLIENT_FLAGS]) {
                u64 flags = nla_get_u64(info->attrs[NBD_ATTR_CLIENT_FLAGS]);
                if (flags & NBD_CFLAG_DESTROY_ON_DISCONNECT) {
                        if (!test_and_set_bit(NBD_DESTROY_ON_DISCONNECT,
                                              &nbd->flags))
                                put_dev = true;
                } else {
                        if (test_and_clear_bit(NBD_DESTROY_ON_DISCONNECT,
                                               &nbd->flags))
                                refcount_inc(&nbd->refs);
                }

                if (flags & NBD_CFLAG_DISCONNECT_ON_CLOSE) {
                        set_bit(NBD_RT_DISCONNECT_ON_CLOSE,
                                        &config->runtime_flags);
                } else {
                        clear_bit(NBD_RT_DISCONNECT_ON_CLOSE,
                                        &config->runtime_flags);
                }
        }

        if (info->attrs[NBD_ATTR_SOCKETS]) {
                struct nlattr *attr;
                int rem, fd;

                nla_for_each_nested(attr, info->attrs[NBD_ATTR_SOCKETS],
                                    rem) {
                        struct nlattr *socks[NBD_SOCK_MAX+1];

                        if (nla_type(attr) != NBD_SOCK_ITEM) {
                                pr_err("socks must be embedded in a SOCK_ITEM attr\n");
                                ret = -EINVAL;
                                goto out;
                        }
                        ret = nla_parse_nested_deprecated(socks, NBD_SOCK_MAX,
                                                          attr,
                                                          nbd_sock_policy,
                                                          info->extack);
                        if (ret != 0) {
                                pr_err("error processing sock list\n");
                                ret = -EINVAL;
                                goto out;
                        }
                        if (!socks[NBD_SOCK_FD])
                                continue;
                        fd = (int)nla_get_u32(socks[NBD_SOCK_FD]);
                        ret = nbd_reconnect_socket(nbd, fd);
                        if (ret) {
                                if (ret == -ENOSPC)
                                        ret = 0;
                                goto out;
                        }
                        dev_info(nbd_to_dev(nbd), "reconnected socket\n");
                }
        }
out:
        mutex_unlock(&nbd->config_lock);
        nbd_config_put(nbd);
        nbd_put(nbd);
        if (put_dev)
                nbd_put(nbd);
        return ret;
}

static const struct genl_small_ops nbd_connect_genl_ops[] = {
        {
                .cmd        = NBD_CMD_CONNECT,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .doit        = nbd_genl_connect,
        },
        {
                .cmd        = NBD_CMD_DISCONNECT,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .doit        = nbd_genl_disconnect,
        },
        {
                .cmd        = NBD_CMD_RECONFIGURE,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .doit        = nbd_genl_reconfigure,
        },
        {
                .cmd        = NBD_CMD_STATUS,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .doit        = nbd_genl_status,
        },
};

static const struct genl_multicast_group nbd_mcast_grps[] = {
        { .name = NBD_GENL_MCAST_GROUP_NAME, },
};

static struct genl_family nbd_genl_family __ro_after_init = {
        .hdrsize        = 0,
        .name                = NBD_GENL_FAMILY_NAME,
        .version        = NBD_GENL_VERSION,
        .module                = THIS_MODULE,
        .small_ops        = nbd_connect_genl_ops,
        .n_small_ops        = ARRAY_SIZE(nbd_connect_genl_ops),
        .resv_start_op        = NBD_CMD_STATUS + 1,
        .maxattr        = NBD_ATTR_MAX,
        .netnsok        = 1,
        .policy = nbd_attr_policy,
        .mcgrps                = nbd_mcast_grps,
        .n_mcgrps        = ARRAY_SIZE(nbd_mcast_grps),
};
MODULE_ALIAS_GENL_FAMILY(NBD_GENL_FAMILY_NAME);

static int populate_nbd_status(struct nbd_device *nbd, struct sk_buff *reply)
{
        struct nlattr *dev_opt;
        u8 connected = 0;
        int ret;

        /* This is a little racey, but for status it's ok.  The
         * reason we don't take a ref here is because we can't
         * take a ref in the index == -1 case as we would need
         * to put under the nbd_index_mutex, which could
         * deadlock if we are configured to remove ourselves
         * once we're disconnected.
         */
        if (refcount_read(&nbd->config_refs))
                connected = 1;
        dev_opt = nla_nest_start_noflag(reply, NBD_DEVICE_ITEM);
        if (!dev_opt)
                return -EMSGSIZE;
        ret = nla_put_u32(reply, NBD_DEVICE_INDEX, nbd->index);
        if (ret)
                return -EMSGSIZE;
        ret = nla_put_u8(reply, NBD_DEVICE_CONNECTED,
                         connected);
        if (ret)
                return -EMSGSIZE;
        nla_nest_end(reply, dev_opt);
        return 0;
}

static int status_cb(int id, void *ptr, void *data)
{
        struct nbd_device *nbd = ptr;
        return populate_nbd_status(nbd, (struct sk_buff *)data);
}

static int nbd_genl_status(struct sk_buff *skb, struct genl_info *info)
{
        struct nlattr *dev_list;
        struct sk_buff *reply;
        void *reply_head;
        size_t msg_size;
        int index = -1;
        int ret = -ENOMEM;

        if (info->attrs[NBD_ATTR_INDEX])
                index = nla_get_u32(info->attrs[NBD_ATTR_INDEX]);

        mutex_lock(&nbd_index_mutex);

        msg_size = nla_total_size(nla_attr_size(sizeof(u32)) +
                                  nla_attr_size(sizeof(u8)));
        msg_size *= (index == -1) ? nbd_total_devices : 1;

        reply = genlmsg_new(msg_size, GFP_KERNEL);
        if (!reply)
                goto out;
        reply_head = genlmsg_put_reply(reply, info, &nbd_genl_family, 0,
                                       NBD_CMD_STATUS);
        if (!reply_head) {
                nlmsg_free(reply);
                goto out;
        }

        dev_list = nla_nest_start_noflag(reply, NBD_ATTR_DEVICE_LIST);
        if (!dev_list) {
                nlmsg_free(reply);
                ret = -EMSGSIZE;
                goto out;
        }

        if (index == -1) {
                ret = idr_for_each(&nbd_index_idr, &status_cb, reply);
                if (ret) {
                        nlmsg_free(reply);
                        goto out;
                }
        } else {
                struct nbd_device *nbd;
                nbd = idr_find(&nbd_index_idr, index);
                if (nbd) {
                        ret = populate_nbd_status(nbd, reply);
                        if (ret) {
                                nlmsg_free(reply);
                                goto out;
                        }
                }
        }
        nla_nest_end(reply, dev_list);
        genlmsg_end(reply, reply_head);
        ret = genlmsg_reply(reply, info);
out:
        mutex_unlock(&nbd_index_mutex);
        return ret;
}

static void nbd_connect_reply(struct genl_info *info, int index)
{
        struct sk_buff *skb;
        void *msg_head;
        int ret;

        skb = genlmsg_new(nla_total_size(sizeof(u32)), GFP_KERNEL);
        if (!skb)
                return;
        msg_head = genlmsg_put_reply(skb, info, &nbd_genl_family, 0,
                                     NBD_CMD_CONNECT);
        if (!msg_head) {
                nlmsg_free(skb);
                return;
        }
        ret = nla_put_u32(skb, NBD_ATTR_INDEX, index);
        if (ret) {
                nlmsg_free(skb);
                return;
        }
        genlmsg_end(skb, msg_head);
        genlmsg_reply(skb, info);
}

static void nbd_mcast_index(int index)
{
        struct sk_buff *skb;
        void *msg_head;
        int ret;

        skb = genlmsg_new(nla_total_size(sizeof(u32)), GFP_KERNEL);
        if (!skb)
                return;
        msg_head = genlmsg_put(skb, 0, 0, &nbd_genl_family, 0,
                                     NBD_CMD_LINK_DEAD);
        if (!msg_head) {
                nlmsg_free(skb);
                return;
        }
        ret = nla_put_u32(skb, NBD_ATTR_INDEX, index);
        if (ret) {
                nlmsg_free(skb);
                return;
        }
        genlmsg_end(skb, msg_head);
        genlmsg_multicast(&nbd_genl_family, skb, 0, 0, GFP_KERNEL);
}

static void nbd_dead_link_work(struct work_struct *work)
{
        struct link_dead_args *args = container_of(work, struct link_dead_args,
                                                   work);
        nbd_mcast_index(args->index);
        kfree(args);
}

static int __init nbd_init(void)
{
        int i;

        BUILD_BUG_ON(sizeof(struct nbd_request) != 28);

        if (max_part < 0) {
                pr_err("max_part must be >= 0\n");
                return -EINVAL;
        }

        part_shift = 0;
        if (max_part > 0) {
                part_shift = fls(max_part);

                /*
                 * Adjust max_part according to part_shift as it is exported
                 * to user space so that user can know the max number of
                 * partition kernel should be able to manage.
                 *
                 * Note that -1 is required because partition 0 is reserved
                 * for the whole disk.
                 */
                max_part = (1UL << part_shift) - 1;
        }

        if ((1UL << part_shift) > DISK_MAX_PARTS)
                return -EINVAL;

        if (nbds_max > 1UL << (MINORBITS - part_shift))
                return -EINVAL;

        if (register_blkdev(NBD_MAJOR, "nbd"))
                return -EIO;

        nbd_del_wq = alloc_workqueue("nbd-del", WQ_UNBOUND, 0);
        if (!nbd_del_wq) {
                unregister_blkdev(NBD_MAJOR, "nbd");
                return -ENOMEM;
        }

        if (genl_register_family(&nbd_genl_family)) {
                destroy_workqueue(nbd_del_wq);
                unregister_blkdev(NBD_MAJOR, "nbd");
                return -EINVAL;
        }
        nbd_dbg_init();

        for (i = 0; i < nbds_max; i++)
                nbd_dev_add(i, 1);
        return 0;
}

static int nbd_exit_cb(int id, void *ptr, void *data)
{
        struct list_head *list = (struct list_head *)data;
        struct nbd_device *nbd = ptr;

        /* Skip nbd that is being removed asynchronously */
        if (refcount_read(&nbd->refs))
                list_add_tail(&nbd->list, list);

        return 0;
}

static void __exit nbd_cleanup(void)
{
        struct nbd_device *nbd;
        LIST_HEAD(del_list);

        /*
         * Unregister netlink interface prior to waiting
         * for the completion of netlink commands.
         */
        genl_unregister_family(&nbd_genl_family);

        nbd_dbg_close();

        mutex_lock(&nbd_index_mutex);
        idr_for_each(&nbd_index_idr, &nbd_exit_cb, &del_list);
        mutex_unlock(&nbd_index_mutex);

        while (!list_empty(&del_list)) {
                nbd = list_first_entry(&del_list, struct nbd_device, list);
                list_del_init(&nbd->list);
                if (refcount_read(&nbd->config_refs))
                        pr_err("possibly leaking nbd_config (ref %d)\n",
                                        refcount_read(&nbd->config_refs));
                if (refcount_read(&nbd->refs) != 1)
                        pr_err("possibly leaking a device\n");
                nbd_put(nbd);
        }

        /* Also wait for nbd_dev_remove_work() completes */
        destroy_workqueue(nbd_del_wq);

        idr_destroy(&nbd_index_idr);
        unregister_blkdev(NBD_MAJOR, "nbd");
}

module_init(nbd_init);
module_exit(nbd_cleanup);

MODULE_DESCRIPTION("Network Block Device");
MODULE_LICENSE("GPL");

module_param(nbds_max, int, 0444);
MODULE_PARM_DESC(nbds_max, "number of network block devices to initialize (default: 16)");
module_param(max_part, int, 0444);
MODULE_PARM_DESC(max_part, "number of partitions per device (default: 16)");










































































































































































































































































































































































































































































































































































































































































































































































































































































































































    5 















   10 

































































































































































































































































































































































































































































































































































































































































































































































































































    5 







    5 


    5 







    5 















































   11 



    6 



   11 

   10 
   10 
   10 



   10 







    1 
    6 
















    6 
    6 










    6 




    1 












    1 




















    6 

















    5 








    1 




   17 





















   11 





    6 

    6 





















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2015 Nicira, Inc.
 */

#include <linux/module.h>
#include <linux/openvswitch.h>
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/sctp.h>
#include <linux/static_key.h>
#include <linux/string_helpers.h>
#include <net/ip.h>
#include <net/genetlink.h>
#include <net/netfilter/nf_conntrack_core.h>
#include <net/netfilter/nf_conntrack_count.h>
#include <net/netfilter/nf_conntrack_helper.h>
#include <net/netfilter/nf_conntrack_labels.h>
#include <net/netfilter/nf_conntrack_seqadj.h>
#include <net/netfilter/nf_conntrack_timeout.h>
#include <net/netfilter/nf_conntrack_zones.h>
#include <net/netfilter/ipv6/nf_defrag_ipv6.h>
#include <net/ipv6_frag.h>

#if IS_ENABLED(CONFIG_NF_NAT)
#include <net/netfilter/nf_nat.h>
#endif

#include <net/netfilter/nf_conntrack_act_ct.h>

#include "datapath.h"
#include "drop.h"
#include "conntrack.h"
#include "flow.h"
#include "flow_netlink.h"

struct ovs_ct_len_tbl {
        int maxlen;
        int minlen;
};

/* Metadata mark for masked write to conntrack mark */
struct md_mark {
        u32 value;
        u32 mask;
};

/* Metadata label for masked write to conntrack label. */
struct md_labels {
        struct ovs_key_ct_labels value;
        struct ovs_key_ct_labels mask;
};

enum ovs_ct_nat {
        OVS_CT_NAT = 1 << 0,     /* NAT for committed connections only. */
        OVS_CT_SRC_NAT = 1 << 1, /* Source NAT for NEW connections. */
        OVS_CT_DST_NAT = 1 << 2, /* Destination NAT for NEW connections. */
};

/* Conntrack action context for execution. */
struct ovs_conntrack_info {
        struct nf_conntrack_helper *helper;
        struct nf_conntrack_zone zone;
        struct nf_conn *ct;
        u8 commit : 1;
        u8 nat : 3;                 /* enum ovs_ct_nat */
        u8 force : 1;
        u8 have_eventmask : 1;
        u16 family;
        u32 eventmask;              /* Mask of 1 << IPCT_*. */
        struct md_mark mark;
        struct md_labels labels;
        char timeout[CTNL_TIMEOUT_NAME_MAX];
        struct nf_ct_timeout *nf_ct_timeout;
#if IS_ENABLED(CONFIG_NF_NAT)
        struct nf_nat_range2 range;  /* Only present for SRC NAT and DST NAT. */
#endif
};

#if        IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT)
#define OVS_CT_LIMIT_UNLIMITED        0
#define OVS_CT_LIMIT_DEFAULT OVS_CT_LIMIT_UNLIMITED
#define CT_LIMIT_HASH_BUCKETS 512
static DEFINE_STATIC_KEY_FALSE(ovs_ct_limit_enabled);

struct ovs_ct_limit {
        /* Elements in ovs_ct_limit_info->limits hash table */
        struct hlist_node hlist_node;
        struct rcu_head rcu;
        u16 zone;
        u32 limit;
};

struct ovs_ct_limit_info {
        u32 default_limit;
        struct hlist_head *limits;
        struct nf_conncount_data *data;
};

static const struct nla_policy ct_limit_policy[OVS_CT_LIMIT_ATTR_MAX + 1] = {
        [OVS_CT_LIMIT_ATTR_ZONE_LIMIT] = { .type = NLA_NESTED, },
};
#endif

static bool labels_nonzero(const struct ovs_key_ct_labels *labels);

static void __ovs_ct_free_action(struct ovs_conntrack_info *ct_info);

static u16 key_to_nfproto(const struct sw_flow_key *key)
{
        switch (ntohs(key->eth.type)) {
        case ETH_P_IP:
                return NFPROTO_IPV4;
        case ETH_P_IPV6:
                return NFPROTO_IPV6;
        default:
                return NFPROTO_UNSPEC;
        }
}

/* Map SKB connection state into the values used by flow definition. */
static u8 ovs_ct_get_state(enum ip_conntrack_info ctinfo)
{
        u8 ct_state = OVS_CS_F_TRACKED;

        switch (ctinfo) {
        case IP_CT_ESTABLISHED_REPLY:
        case IP_CT_RELATED_REPLY:
                ct_state |= OVS_CS_F_REPLY_DIR;
                break;
        default:
                break;
        }

        switch (ctinfo) {
        case IP_CT_ESTABLISHED:
        case IP_CT_ESTABLISHED_REPLY:
                ct_state |= OVS_CS_F_ESTABLISHED;
                break;
        case IP_CT_RELATED:
        case IP_CT_RELATED_REPLY:
                ct_state |= OVS_CS_F_RELATED;
                break;
        case IP_CT_NEW:
                ct_state |= OVS_CS_F_NEW;
                break;
        default:
                break;
        }

        return ct_state;
}

static u32 ovs_ct_get_mark(const struct nf_conn *ct)
{
#if IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)
        return ct ? READ_ONCE(ct->mark) : 0;
#else
        return 0;
#endif
}

/* Guard against conntrack labels max size shrinking below 128 bits. */
#if NF_CT_LABELS_MAX_SIZE < 16
#error NF_CT_LABELS_MAX_SIZE must be at least 16 bytes
#endif

static void ovs_ct_get_labels(const struct nf_conn *ct,
                              struct ovs_key_ct_labels *labels)
{
        struct nf_conn_labels *cl = NULL;

        if (ct) {
                if (ct->master && !nf_ct_is_confirmed(ct))
                        ct = ct->master;
                cl = nf_ct_labels_find(ct);
        }
        if (cl)
                memcpy(labels, cl->bits, OVS_CT_LABELS_LEN);
        else
                memset(labels, 0, OVS_CT_LABELS_LEN);
}

static void __ovs_ct_update_key_orig_tp(struct sw_flow_key *key,
                                        const struct nf_conntrack_tuple *orig,
                                        u8 icmp_proto)
{
        key->ct_orig_proto = orig->dst.protonum;
        if (orig->dst.protonum == icmp_proto) {
                key->ct.orig_tp.src = htons(orig->dst.u.icmp.type);
                key->ct.orig_tp.dst = htons(orig->dst.u.icmp.code);
        } else {
                key->ct.orig_tp.src = orig->src.u.all;
                key->ct.orig_tp.dst = orig->dst.u.all;
        }
}

static void __ovs_ct_update_key(struct sw_flow_key *key, u8 state,
                                const struct nf_conntrack_zone *zone,
                                const struct nf_conn *ct)
{
        key->ct_state = state;
        key->ct_zone = zone->id;
        key->ct.mark = ovs_ct_get_mark(ct);
        ovs_ct_get_labels(ct, &key->ct.labels);

        if (ct) {
                const struct nf_conntrack_tuple *orig;

                /* Use the master if we have one. */
                if (ct->master)
                        ct = ct->master;
                orig = &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple;

                /* IP version must match with the master connection. */
                if (key->eth.type == htons(ETH_P_IP) &&
                    nf_ct_l3num(ct) == NFPROTO_IPV4) {
                        key->ipv4.ct_orig.src = orig->src.u3.ip;
                        key->ipv4.ct_orig.dst = orig->dst.u3.ip;
                        __ovs_ct_update_key_orig_tp(key, orig, IPPROTO_ICMP);
                        return;
                } else if (key->eth.type == htons(ETH_P_IPV6) &&
                           !sw_flow_key_is_nd(key) &&
                           nf_ct_l3num(ct) == NFPROTO_IPV6) {
                        key->ipv6.ct_orig.src = orig->src.u3.in6;
                        key->ipv6.ct_orig.dst = orig->dst.u3.in6;
                        __ovs_ct_update_key_orig_tp(key, orig, NEXTHDR_ICMP);
                        return;
                }
        }
        /* Clear 'ct_orig_proto' to mark the non-existence of conntrack
         * original direction key fields.
         */
        key->ct_orig_proto = 0;
}

/* Update 'key' based on skb->_nfct.  If 'post_ct' is true, then OVS has
 * previously sent the packet to conntrack via the ct action.  If
 * 'keep_nat_flags' is true, the existing NAT flags retained, else they are
 * initialized from the connection status.
 */
static void ovs_ct_update_key(const struct sk_buff *skb,
                              const struct ovs_conntrack_info *info,
                              struct sw_flow_key *key, bool post_ct,
                              bool keep_nat_flags)
{
        const struct nf_conntrack_zone *zone = &nf_ct_zone_dflt;
        enum ip_conntrack_info ctinfo;
        struct nf_conn *ct;
        u8 state = 0;

        ct = nf_ct_get(skb, &ctinfo);
        if (ct) {
                state = ovs_ct_get_state(ctinfo);
                /* All unconfirmed entries are NEW connections. */
                if (!nf_ct_is_confirmed(ct))
                        state |= OVS_CS_F_NEW;
                /* OVS persists the related flag for the duration of the
                 * connection.
                 */
                if (ct->master)
                        state |= OVS_CS_F_RELATED;
                if (keep_nat_flags) {
                        state |= key->ct_state & OVS_CS_F_NAT_MASK;
                } else {
                        if (ct->status & IPS_SRC_NAT)
                                state |= OVS_CS_F_SRC_NAT;
                        if (ct->status & IPS_DST_NAT)
                                state |= OVS_CS_F_DST_NAT;
                }
                zone = nf_ct_zone(ct);
        } else if (post_ct) {
                state = OVS_CS_F_TRACKED | OVS_CS_F_INVALID;
                if (info)
                        zone = &info->zone;
        }
        __ovs_ct_update_key(key, state, zone, ct);
}

/* This is called to initialize CT key fields possibly coming in from the local
 * stack.
 */
void ovs_ct_fill_key(const struct sk_buff *skb,
                     struct sw_flow_key *key,
                     bool post_ct)
{
        ovs_ct_update_key(skb, NULL, key, post_ct, false);
}

int ovs_ct_put_key(const struct sw_flow_key *swkey,
                   const struct sw_flow_key *output, struct sk_buff *skb)
{
        if (nla_put_u32(skb, OVS_KEY_ATTR_CT_STATE, output->ct_state))
                return -EMSGSIZE;

        if (IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES) &&
            nla_put_u16(skb, OVS_KEY_ATTR_CT_ZONE, output->ct_zone))
                return -EMSGSIZE;

        if (IS_ENABLED(CONFIG_NF_CONNTRACK_MARK) &&
            nla_put_u32(skb, OVS_KEY_ATTR_CT_MARK, output->ct.mark))
                return -EMSGSIZE;

        if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS) &&
            nla_put(skb, OVS_KEY_ATTR_CT_LABELS, sizeof(output->ct.labels),
                    &output->ct.labels))
                return -EMSGSIZE;

        if (swkey->ct_orig_proto) {
                if (swkey->eth.type == htons(ETH_P_IP)) {
                        struct ovs_key_ct_tuple_ipv4 orig;

                        memset(&orig, 0, sizeof(orig));
                        orig.ipv4_src = output->ipv4.ct_orig.src;
                        orig.ipv4_dst = output->ipv4.ct_orig.dst;
                        orig.src_port = output->ct.orig_tp.src;
                        orig.dst_port = output->ct.orig_tp.dst;
                        orig.ipv4_proto = output->ct_orig_proto;

                        if (nla_put(skb, OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV4,
                                    sizeof(orig), &orig))
                                return -EMSGSIZE;
                } else if (swkey->eth.type == htons(ETH_P_IPV6)) {
                        struct ovs_key_ct_tuple_ipv6 orig;

                        memset(&orig, 0, sizeof(orig));
                        memcpy(orig.ipv6_src, output->ipv6.ct_orig.src.s6_addr32,
                               sizeof(orig.ipv6_src));
                        memcpy(orig.ipv6_dst, output->ipv6.ct_orig.dst.s6_addr32,
                               sizeof(orig.ipv6_dst));
                        orig.src_port = output->ct.orig_tp.src;
                        orig.dst_port = output->ct.orig_tp.dst;
                        orig.ipv6_proto = output->ct_orig_proto;

                        if (nla_put(skb, OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV6,
                                    sizeof(orig), &orig))
                                return -EMSGSIZE;
                }
        }

        return 0;
}

static int ovs_ct_set_mark(struct nf_conn *ct, struct sw_flow_key *key,
                           u32 ct_mark, u32 mask)
{
#if IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)
        u32 new_mark;

        new_mark = ct_mark | (READ_ONCE(ct->mark) & ~(mask));
        if (READ_ONCE(ct->mark) != new_mark) {
                WRITE_ONCE(ct->mark, new_mark);
                if (nf_ct_is_confirmed(ct))
                        nf_conntrack_event_cache(IPCT_MARK, ct);
                key->ct.mark = new_mark;
        }

        return 0;
#else
        return -ENOTSUPP;
#endif
}

static struct nf_conn_labels *ovs_ct_get_conn_labels(struct nf_conn *ct)
{
        struct nf_conn_labels *cl;

        cl = nf_ct_labels_find(ct);
        if (!cl) {
                nf_ct_labels_ext_add(ct);
                cl = nf_ct_labels_find(ct);
        }

        return cl;
}

/* Initialize labels for a new, yet to be committed conntrack entry.  Note that
 * since the new connection is not yet confirmed, and thus no-one else has
 * access to it's labels, we simply write them over.
 */
static int ovs_ct_init_labels(struct nf_conn *ct, struct sw_flow_key *key,
                              const struct ovs_key_ct_labels *labels,
                              const struct ovs_key_ct_labels *mask)
{
        struct nf_conn_labels *cl, *master_cl;
        bool have_mask = labels_nonzero(mask);

        /* Inherit master's labels to the related connection? */
        master_cl = ct->master ? nf_ct_labels_find(ct->master) : NULL;

        if (!master_cl && !have_mask)
                return 0;   /* Nothing to do. */

        cl = ovs_ct_get_conn_labels(ct);
        if (!cl)
                return -ENOSPC;

        /* Inherit the master's labels, if any. */
        if (master_cl)
                *cl = *master_cl;

        if (have_mask) {
                u32 *dst = (u32 *)cl->bits;
                int i;

                for (i = 0; i < OVS_CT_LABELS_LEN_32; i++)
                        dst[i] = (dst[i] & ~mask->ct_labels_32[i]) |
                                (labels->ct_labels_32[i]
                                 & mask->ct_labels_32[i]);
        }

        /* Labels are included in the IPCTNL_MSG_CT_NEW event only if the
         * IPCT_LABEL bit is set in the event cache.
         */
        nf_conntrack_event_cache(IPCT_LABEL, ct);

        memcpy(&key->ct.labels, cl->bits, OVS_CT_LABELS_LEN);

        return 0;
}

static int ovs_ct_set_labels(struct nf_conn *ct, struct sw_flow_key *key,
                             const struct ovs_key_ct_labels *labels,
                             const struct ovs_key_ct_labels *mask)
{
        struct nf_conn_labels *cl;
        int err;

        cl = ovs_ct_get_conn_labels(ct);
        if (!cl)
                return -ENOSPC;

        err = nf_connlabels_replace(ct, labels->ct_labels_32,
                                    mask->ct_labels_32,
                                    OVS_CT_LABELS_LEN_32);
        if (err)
                return err;

        memcpy(&key->ct.labels, cl->bits, OVS_CT_LABELS_LEN);

        return 0;
}

static int ovs_ct_handle_fragments(struct net *net, struct sw_flow_key *key,
                                   u16 zone, int family, struct sk_buff *skb)
{
        struct ovs_skb_cb ovs_cb = *OVS_CB(skb);
        int err;

        err = nf_ct_handle_fragments(net, skb, zone, family, &key->ip.proto, &ovs_cb.mru);
        if (err)
                return err;

        /* The key extracted from the fragment that completed this datagram
         * likely didn't have an L4 header, so regenerate it.
         */
        ovs_flow_key_update_l3l4(skb, key);
        key->ip.frag = OVS_FRAG_TYPE_NONE;
        *OVS_CB(skb) = ovs_cb;

        return 0;
}

/* This replicates logic from nf_conntrack_core.c that is not exported. */
static enum ip_conntrack_info
ovs_ct_get_info(const struct nf_conntrack_tuple_hash *h)
{
        const struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);

        if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY)
                return IP_CT_ESTABLISHED_REPLY;
        /* Once we've had two way comms, always ESTABLISHED. */
        if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status))
                return IP_CT_ESTABLISHED;
        if (test_bit(IPS_EXPECTED_BIT, &ct->status))
                return IP_CT_RELATED;
        return IP_CT_NEW;
}

/* Find an existing connection which this packet belongs to without
 * re-attributing statistics or modifying the connection state.  This allows an
 * skb->_nfct lost due to an upcall to be recovered during actions execution.
 *
 * Must be called with rcu_read_lock.
 *
 * On success, populates skb->_nfct and returns the connection.  Returns NULL
 * if there is no existing entry.
 */
static struct nf_conn *
ovs_ct_find_existing(struct net *net, const struct nf_conntrack_zone *zone,
                     u8 l3num, struct sk_buff *skb, bool natted)
{
        struct nf_conntrack_tuple tuple;
        struct nf_conntrack_tuple_hash *h;
        struct nf_conn *ct;

        if (!nf_ct_get_tuplepr(skb, skb_network_offset(skb), l3num,
                               net, &tuple)) {
                pr_debug("ovs_ct_find_existing: Can't get tuple\n");
                return NULL;
        }

        /* Must invert the tuple if skb has been transformed by NAT. */
        if (natted) {
                struct nf_conntrack_tuple inverse;

                if (!nf_ct_invert_tuple(&inverse, &tuple)) {
                        pr_debug("ovs_ct_find_existing: Inversion failed!\n");
                        return NULL;
                }
                tuple = inverse;
        }

        /* look for tuple match */
        h = nf_conntrack_find_get(net, zone, &tuple);
        if (!h)
                return NULL;   /* Not found. */

        ct = nf_ct_tuplehash_to_ctrack(h);

        /* Inverted packet tuple matches the reverse direction conntrack tuple,
         * select the other tuplehash to get the right 'ctinfo' bits for this
         * packet.
         */
        if (natted)
                h = &ct->tuplehash[!h->tuple.dst.dir];

        nf_ct_set(skb, ct, ovs_ct_get_info(h));
        return ct;
}

static
struct nf_conn *ovs_ct_executed(struct net *net,
                                const struct sw_flow_key *key,
                                const struct ovs_conntrack_info *info,
                                struct sk_buff *skb,
                                bool *ct_executed)
{
        struct nf_conn *ct = NULL;

        /* If no ct, check if we have evidence that an existing conntrack entry
         * might be found for this skb.  This happens when we lose a skb->_nfct
         * due to an upcall, or if the direction is being forced.  If the
         * connection was not confirmed, it is not cached and needs to be run
         * through conntrack again.
         */
        *ct_executed = (key->ct_state & OVS_CS_F_TRACKED) &&
                       !(key->ct_state & OVS_CS_F_INVALID) &&
                       (key->ct_zone == info->zone.id);

        if (*ct_executed || (!key->ct_state && info->force)) {
                ct = ovs_ct_find_existing(net, &info->zone, info->family, skb,
                                          !!(key->ct_state &
                                          OVS_CS_F_NAT_MASK));
        }

        return ct;
}

/* Determine whether skb->_nfct is equal to the result of conntrack lookup. */
static bool skb_nfct_cached(struct net *net,
                            const struct sw_flow_key *key,
                            const struct ovs_conntrack_info *info,
                            struct sk_buff *skb)
{
        enum ip_conntrack_info ctinfo;
        struct nf_conn *ct;
        bool ct_executed = true;

        ct = nf_ct_get(skb, &ctinfo);
        if (!ct)
                ct = ovs_ct_executed(net, key, info, skb, &ct_executed);

        if (ct)
                nf_ct_get(skb, &ctinfo);
        else
                return false;

        if (!net_eq(net, read_pnet(&ct->ct_net)))
                return false;
        if (!nf_ct_zone_equal_any(info->ct, nf_ct_zone(ct)))
                return false;
        if (info->helper) {
                struct nf_conn_help *help;

                help = nf_ct_ext_find(ct, NF_CT_EXT_HELPER);
                if (help && rcu_access_pointer(help->helper) != info->helper)
                        return false;
        }
        if (info->nf_ct_timeout) {
                struct nf_conn_timeout *timeout_ext;

                timeout_ext = nf_ct_timeout_find(ct);
                if (!timeout_ext || info->nf_ct_timeout !=
                    rcu_dereference(timeout_ext->timeout))
                        return false;
        }
        /* Force conntrack entry direction to the current packet? */
        if (info->force && CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL) {
                /* Delete the conntrack entry if confirmed, else just release
                 * the reference.
                 */
                if (nf_ct_is_confirmed(ct))
                        nf_ct_delete(ct, 0, 0);

                nf_ct_put(ct);
                nf_ct_set(skb, NULL, 0);
                return false;
        }

        return ct_executed;
}

#if IS_ENABLED(CONFIG_NF_NAT)
static void ovs_nat_update_key(struct sw_flow_key *key,
                               const struct sk_buff *skb,
                               enum nf_nat_manip_type maniptype)
{
        if (maniptype == NF_NAT_MANIP_SRC) {
                __be16 src;

                key->ct_state |= OVS_CS_F_SRC_NAT;
                if (key->eth.type == htons(ETH_P_IP))
                        key->ipv4.addr.src = ip_hdr(skb)->saddr;
                else if (key->eth.type == htons(ETH_P_IPV6))
                        memcpy(&key->ipv6.addr.src, &ipv6_hdr(skb)->saddr,
                               sizeof(key->ipv6.addr.src));
                else
                        return;

                if (key->ip.proto == IPPROTO_UDP)
                        src = udp_hdr(skb)->source;
                else if (key->ip.proto == IPPROTO_TCP)
                        src = tcp_hdr(skb)->source;
                else if (key->ip.proto == IPPROTO_SCTP)
                        src = sctp_hdr(skb)->source;
                else
                        return;

                key->tp.src = src;
        } else {
                __be16 dst;

                key->ct_state |= OVS_CS_F_DST_NAT;
                if (key->eth.type == htons(ETH_P_IP))
                        key->ipv4.addr.dst = ip_hdr(skb)->daddr;
                else if (key->eth.type == htons(ETH_P_IPV6))
                        memcpy(&key->ipv6.addr.dst, &ipv6_hdr(skb)->daddr,
                               sizeof(key->ipv6.addr.dst));
                else
                        return;

                if (key->ip.proto == IPPROTO_UDP)
                        dst = udp_hdr(skb)->dest;
                else if (key->ip.proto == IPPROTO_TCP)
                        dst = tcp_hdr(skb)->dest;
                else if (key->ip.proto == IPPROTO_SCTP)
                        dst = sctp_hdr(skb)->dest;
                else
                        return;

                key->tp.dst = dst;
        }
}

/* Returns NF_DROP if the packet should be dropped, NF_ACCEPT otherwise. */
static int ovs_ct_nat(struct net *net, struct sw_flow_key *key,
                      const struct ovs_conntrack_info *info,
                      struct sk_buff *skb, struct nf_conn *ct,
                      enum ip_conntrack_info ctinfo)
{
        int err, action = 0;

        if (!(info->nat & OVS_CT_NAT))
                return NF_ACCEPT;
        if (info->nat & OVS_CT_SRC_NAT)
                action |= BIT(NF_NAT_MANIP_SRC);
        if (info->nat & OVS_CT_DST_NAT)
                action |= BIT(NF_NAT_MANIP_DST);

        err = nf_ct_nat(skb, ct, ctinfo, &action, &info->range, info->commit);
        if (err != NF_ACCEPT)
                return err;

        if (action & BIT(NF_NAT_MANIP_SRC))
                ovs_nat_update_key(key, skb, NF_NAT_MANIP_SRC);
        if (action & BIT(NF_NAT_MANIP_DST))
                ovs_nat_update_key(key, skb, NF_NAT_MANIP_DST);

        return err;
}
#else /* !CONFIG_NF_NAT */
static int ovs_ct_nat(struct net *net, struct sw_flow_key *key,
                      const struct ovs_conntrack_info *info,
                      struct sk_buff *skb, struct nf_conn *ct,
                      enum ip_conntrack_info ctinfo)
{
        return NF_ACCEPT;
}
#endif

static int verdict_to_errno(unsigned int verdict)
{
        switch (verdict & NF_VERDICT_MASK) {
        case NF_ACCEPT:
                return 0;
        case NF_DROP:
                return -EINVAL;
        case NF_STOLEN:
                return -EINPROGRESS;
        default:
                break;
        }

        return -EINVAL;
}

/* Pass 'skb' through conntrack in 'net', using zone configured in 'info', if
 * not done already.  Update key with new CT state after passing the packet
 * through conntrack.
 * Note that if the packet is deemed invalid by conntrack, skb->_nfct will be
 * set to NULL and 0 will be returned.
 */
static int __ovs_ct_lookup(struct net *net, struct sw_flow_key *key,
                           const struct ovs_conntrack_info *info,
                           struct sk_buff *skb)
{
        /* If we are recirculating packets to match on conntrack fields and
         * committing with a separate conntrack action,  then we don't need to
         * actually run the packet through conntrack twice unless it's for a
         * different zone.
         */
        bool cached = skb_nfct_cached(net, key, info, skb);
        enum ip_conntrack_info ctinfo;
        struct nf_conn *ct;

        if (!cached) {
                struct nf_hook_state state = {
                        .hook = NF_INET_PRE_ROUTING,
                        .pf = info->family,
                        .net = net,
                };
                struct nf_conn *tmpl = info->ct;
                int err;

                /* Associate skb with specified zone. */
                if (tmpl) {
                        ct = nf_ct_get(skb, &ctinfo);
                        nf_ct_put(ct);
                        nf_conntrack_get(&tmpl->ct_general);
                        nf_ct_set(skb, tmpl, IP_CT_NEW);
                }

                err = nf_conntrack_in(skb, &state);
                if (err != NF_ACCEPT)
                        return verdict_to_errno(err);

                /* Clear CT state NAT flags to mark that we have not yet done
                 * NAT after the nf_conntrack_in() call.  We can actually clear
                 * the whole state, as it will be re-initialized below.
                 */
                key->ct_state = 0;

                /* Update the key, but keep the NAT flags. */
                ovs_ct_update_key(skb, info, key, true, true);
        }

        ct = nf_ct_get(skb, &ctinfo);
        if (ct) {
                bool add_helper = false;

                /* Packets starting a new connection must be NATted before the
                 * helper, so that the helper knows about the NAT.  We enforce
                 * this by delaying both NAT and helper calls for unconfirmed
                 * connections until the committing CT action.  For later
                 * packets NAT and Helper may be called in either order.
                 *
                 * NAT will be done only if the CT action has NAT, and only
                 * once per packet (per zone), as guarded by the NAT bits in
                 * the key->ct_state.
                 */
                if (info->nat && !(key->ct_state & OVS_CS_F_NAT_MASK) &&
                    (nf_ct_is_confirmed(ct) || info->commit)) {
                        int err = ovs_ct_nat(net, key, info, skb, ct, ctinfo);

                        err = verdict_to_errno(err);
                        if (err)
                                return err;
                }

                /* Userspace may decide to perform a ct lookup without a helper
                 * specified followed by a (recirculate and) commit with one,
                 * or attach a helper in a later commit.  Therefore, for
                 * connections which we will commit, we may need to attach
                 * the helper here.
                 */
                if (!nf_ct_is_confirmed(ct) && info->commit &&
                    info->helper && !nfct_help(ct)) {
                        int err = __nf_ct_try_assign_helper(ct, info->ct,
                                                            GFP_ATOMIC);
                        if (err)
                                return err;
                        add_helper = true;

                        /* helper installed, add seqadj if NAT is required */
                        if (info->nat && !nfct_seqadj(ct)) {
                                if (!nfct_seqadj_ext_add(ct))
                                        return -EINVAL;
                        }
                }

                /* Call the helper only if:
                 * - nf_conntrack_in() was executed above ("!cached") or a
                 *   helper was just attached ("add_helper") for a confirmed
                 *   connection, or
                 * - When committing an unconfirmed connection.
                 */
                if ((nf_ct_is_confirmed(ct) ? !cached || add_helper :
                                              info->commit)) {
                        int err = nf_ct_helper(skb, ct, ctinfo, info->family);

                        err = verdict_to_errno(err);
                        if (err)
                                return err;
                }

                if (nf_ct_protonum(ct) == IPPROTO_TCP &&
                    nf_ct_is_confirmed(ct) && nf_conntrack_tcp_established(ct)) {
                        /* Be liberal for tcp packets so that out-of-window
                         * packets are not marked invalid.
                         */
                        nf_ct_set_tcp_be_liberal(ct);
                }

                nf_conn_act_ct_ext_fill(skb, ct, ctinfo);
        }

        return 0;
}

/* Lookup connection and read fields into key. */
static int ovs_ct_lookup(struct net *net, struct sw_flow_key *key,
                         const struct ovs_conntrack_info *info,
                         struct sk_buff *skb)
{
        struct nf_conn *ct;
        int err;

        err = __ovs_ct_lookup(net, key, info, skb);
        if (err)
                return err;

        ct = (struct nf_conn *)skb_nfct(skb);
        if (ct)
                nf_ct_deliver_cached_events(ct);

        return 0;
}

static bool labels_nonzero(const struct ovs_key_ct_labels *labels)
{
        size_t i;

        for (i = 0; i < OVS_CT_LABELS_LEN_32; i++)
                if (labels->ct_labels_32[i])
                        return true;

        return false;
}

#if        IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT)
static struct hlist_head *ct_limit_hash_bucket(
        const struct ovs_ct_limit_info *info, u16 zone)
{
        return &info->limits[zone & (CT_LIMIT_HASH_BUCKETS - 1)];
}

/* Call with ovs_mutex */
static void ct_limit_set(const struct ovs_ct_limit_info *info,
                         struct ovs_ct_limit *new_ct_limit)
{
        struct ovs_ct_limit *ct_limit;
        struct hlist_head *head;

        head = ct_limit_hash_bucket(info, new_ct_limit->zone);
        hlist_for_each_entry_rcu(ct_limit, head, hlist_node) {
                if (ct_limit->zone == new_ct_limit->zone) {
                        hlist_replace_rcu(&ct_limit->hlist_node,
                                          &new_ct_limit->hlist_node);
                        kfree_rcu(ct_limit, rcu);
                        return;
                }
        }

        hlist_add_head_rcu(&new_ct_limit->hlist_node, head);
}

/* Call with ovs_mutex */
static void ct_limit_del(const struct ovs_ct_limit_info *info, u16 zone)
{
        struct ovs_ct_limit *ct_limit;
        struct hlist_head *head;
        struct hlist_node *n;

        head = ct_limit_hash_bucket(info, zone);
        hlist_for_each_entry_safe(ct_limit, n, head, hlist_node) {
                if (ct_limit->zone == zone) {
                        hlist_del_rcu(&ct_limit->hlist_node);
                        kfree_rcu(ct_limit, rcu);
                        return;
                }
        }
}

/* Call with RCU read lock */
static u32 ct_limit_get(const struct ovs_ct_limit_info *info, u16 zone)
{
        struct ovs_ct_limit *ct_limit;
        struct hlist_head *head;

        head = ct_limit_hash_bucket(info, zone);
        hlist_for_each_entry_rcu(ct_limit, head, hlist_node) {
                if (ct_limit->zone == zone)
                        return ct_limit->limit;
        }

        return info->default_limit;
}

static int ovs_ct_check_limit(struct net *net,
                              const struct ovs_conntrack_info *info,
                              const struct nf_conntrack_tuple *tuple)
{
        struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
        const struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info;
        u32 per_zone_limit, connections;
        u32 conncount_key;

        conncount_key = info->zone.id;

        per_zone_limit = ct_limit_get(ct_limit_info, info->zone.id);
        if (per_zone_limit == OVS_CT_LIMIT_UNLIMITED)
                return 0;

        connections = nf_conncount_count(net, ct_limit_info->data,
                                         &conncount_key, tuple, &info->zone);
        if (connections > per_zone_limit)
                return -ENOMEM;

        return 0;
}
#endif

/* Lookup connection and confirm if unconfirmed. */
static int ovs_ct_commit(struct net *net, struct sw_flow_key *key,
                         const struct ovs_conntrack_info *info,
                         struct sk_buff *skb)
{
        enum ip_conntrack_info ctinfo;
        struct nf_conn *ct;
        int err;

        err = __ovs_ct_lookup(net, key, info, skb);
        if (err)
                return err;

        /* The connection could be invalid, in which case this is a no-op.*/
        ct = nf_ct_get(skb, &ctinfo);
        if (!ct)
                return 0;

#if        IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT)
        if (static_branch_unlikely(&ovs_ct_limit_enabled)) {
                if (!nf_ct_is_confirmed(ct)) {
                        err = ovs_ct_check_limit(net, info,
                                &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
                        if (err) {
                                net_warn_ratelimited("openvswitch: zone: %u "
                                        "exceeds conntrack limit\n",
                                        info->zone.id);
                                return err;
                        }
                }
        }
#endif

        /* Set the conntrack event mask if given.  NEW and DELETE events have
         * their own groups, but the NFNLGRP_CONNTRACK_UPDATE group listener
         * typically would receive many kinds of updates.  Setting the event
         * mask allows those events to be filtered.  The set event mask will
         * remain in effect for the lifetime of the connection unless changed
         * by a further CT action with both the commit flag and the eventmask
         * option. */
        if (info->have_eventmask) {
                struct nf_conntrack_ecache *cache = nf_ct_ecache_find(ct);

                if (cache)
                        cache->ctmask = info->eventmask;
        }

        /* Apply changes before confirming the connection so that the initial
         * conntrack NEW netlink event carries the values given in the CT
         * action.
         */
        if (info->mark.mask) {
                err = ovs_ct_set_mark(ct, key, info->mark.value,
                                      info->mark.mask);
                if (err)
                        return err;
        }
        if (!nf_ct_is_confirmed(ct)) {
                err = ovs_ct_init_labels(ct, key, &info->labels.value,
                                         &info->labels.mask);
                if (err)
                        return err;

                nf_conn_act_ct_ext_add(skb, ct, ctinfo);
        } else if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS) &&
                   labels_nonzero(&info->labels.mask)) {
                err = ovs_ct_set_labels(ct, key, &info->labels.value,
                                        &info->labels.mask);
                if (err)
                        return err;
        }
        /* This will take care of sending queued events even if the connection
         * is already confirmed.
         */
        err = nf_conntrack_confirm(skb);

        return verdict_to_errno(err);
}

/* Returns 0 on success, -EINPROGRESS if 'skb' is stolen, or other nonzero
 * value if 'skb' is freed.
 */
int ovs_ct_execute(struct net *net, struct sk_buff *skb,
                   struct sw_flow_key *key,
                   const struct ovs_conntrack_info *info)
{
        int nh_ofs;
        int err;

        /* The conntrack module expects to be working at L3. */
        nh_ofs = skb_network_offset(skb);
        skb_pull_rcsum(skb, nh_ofs);

        err = nf_ct_skb_network_trim(skb, info->family);
        if (err) {
                kfree_skb(skb);
                return err;
        }

        if (key->ip.frag != OVS_FRAG_TYPE_NONE) {
                err = ovs_ct_handle_fragments(net, key, info->zone.id,
                                              info->family, skb);
                if (err)
                        return err;
        }

        if (info->commit)
                err = ovs_ct_commit(net, key, info, skb);
        else
                err = ovs_ct_lookup(net, key, info, skb);

        /* conntrack core returned NF_STOLEN */
        if (err == -EINPROGRESS)
                return err;

        skb_push_rcsum(skb, nh_ofs);
        if (err)
                ovs_kfree_skb_reason(skb, OVS_DROP_CONNTRACK);
        return err;
}

int ovs_ct_clear(struct sk_buff *skb, struct sw_flow_key *key)
{
        enum ip_conntrack_info ctinfo;
        struct nf_conn *ct;

        ct = nf_ct_get(skb, &ctinfo);

        nf_ct_put(ct);
        nf_ct_set(skb, NULL, IP_CT_UNTRACKED);

        if (key)
                ovs_ct_fill_key(skb, key, false);

        return 0;
}

#if IS_ENABLED(CONFIG_NF_NAT)
static int parse_nat(const struct nlattr *attr,
                     struct ovs_conntrack_info *info, bool log)
{
        struct nlattr *a;
        int rem;
        bool have_ip_max = false;
        bool have_proto_max = false;
        bool ip_vers = (info->family == NFPROTO_IPV6);

        nla_for_each_nested(a, attr, rem) {
                static const int ovs_nat_attr_lens[OVS_NAT_ATTR_MAX + 1][2] = {
                        [OVS_NAT_ATTR_SRC] = {0, 0},
                        [OVS_NAT_ATTR_DST] = {0, 0},
                        [OVS_NAT_ATTR_IP_MIN] = {sizeof(struct in_addr),
                                                 sizeof(struct in6_addr)},
                        [OVS_NAT_ATTR_IP_MAX] = {sizeof(struct in_addr),
                                                 sizeof(struct in6_addr)},
                        [OVS_NAT_ATTR_PROTO_MIN] = {sizeof(u16), sizeof(u16)},
                        [OVS_NAT_ATTR_PROTO_MAX] = {sizeof(u16), sizeof(u16)},
                        [OVS_NAT_ATTR_PERSISTENT] = {0, 0},
                        [OVS_NAT_ATTR_PROTO_HASH] = {0, 0},
                        [OVS_NAT_ATTR_PROTO_RANDOM] = {0, 0},
                };
                int type = nla_type(a);

                if (type > OVS_NAT_ATTR_MAX) {
                        OVS_NLERR(log, "Unknown NAT attribute (type=%d, max=%d)",
                                  type, OVS_NAT_ATTR_MAX);
                        return -EINVAL;
                }

                if (nla_len(a) != ovs_nat_attr_lens[type][ip_vers]) {
                        OVS_NLERR(log, "NAT attribute type %d has unexpected length (%d != %d)",
                                  type, nla_len(a),
                                  ovs_nat_attr_lens[type][ip_vers]);
                        return -EINVAL;
                }

                switch (type) {
                case OVS_NAT_ATTR_SRC:
                case OVS_NAT_ATTR_DST:
                        if (info->nat) {
                                OVS_NLERR(log, "Only one type of NAT may be specified");
                                return -ERANGE;
                        }
                        info->nat |= OVS_CT_NAT;
                        info->nat |= ((type == OVS_NAT_ATTR_SRC)
                                        ? OVS_CT_SRC_NAT : OVS_CT_DST_NAT);
                        break;

                case OVS_NAT_ATTR_IP_MIN:
                        nla_memcpy(&info->range.min_addr, a,
                                   sizeof(info->range.min_addr));
                        info->range.flags |= NF_NAT_RANGE_MAP_IPS;
                        break;

                case OVS_NAT_ATTR_IP_MAX:
                        have_ip_max = true;
                        nla_memcpy(&info->range.max_addr, a,
                                   sizeof(info->range.max_addr));
                        info->range.flags |= NF_NAT_RANGE_MAP_IPS;
                        break;

                case OVS_NAT_ATTR_PROTO_MIN:
                        info->range.min_proto.all = htons(nla_get_u16(a));
                        info->range.flags |= NF_NAT_RANGE_PROTO_SPECIFIED;
                        break;

                case OVS_NAT_ATTR_PROTO_MAX:
                        have_proto_max = true;
                        info->range.max_proto.all = htons(nla_get_u16(a));
                        info->range.flags |= NF_NAT_RANGE_PROTO_SPECIFIED;
                        break;

                case OVS_NAT_ATTR_PERSISTENT:
                        info->range.flags |= NF_NAT_RANGE_PERSISTENT;
                        break;

                case OVS_NAT_ATTR_PROTO_HASH:
                        info->range.flags |= NF_NAT_RANGE_PROTO_RANDOM;
                        break;

                case OVS_NAT_ATTR_PROTO_RANDOM:
                        info->range.flags |= NF_NAT_RANGE_PROTO_RANDOM_FULLY;
                        break;

                default:
                        OVS_NLERR(log, "Unknown nat attribute (%d)", type);
                        return -EINVAL;
                }
        }

        if (rem > 0) {
                OVS_NLERR(log, "NAT attribute has %d unknown bytes", rem);
                return -EINVAL;
        }
        if (!info->nat) {
                /* Do not allow flags if no type is given. */
                if (info->range.flags) {
                        OVS_NLERR(log,
                                  "NAT flags may be given only when NAT range (SRC or DST) is also specified."
                                  );
                        return -EINVAL;
                }
                info->nat = OVS_CT_NAT;   /* NAT existing connections. */
        } else if (!info->commit) {
                OVS_NLERR(log,
                          "NAT attributes may be specified only when CT COMMIT flag is also specified."
                          );
                return -EINVAL;
        }
        /* Allow missing IP_MAX. */
        if (info->range.flags & NF_NAT_RANGE_MAP_IPS && !have_ip_max) {
                memcpy(&info->range.max_addr, &info->range.min_addr,
                       sizeof(info->range.max_addr));
        }
        /* Allow missing PROTO_MAX. */
        if (info->range.flags & NF_NAT_RANGE_PROTO_SPECIFIED &&
            !have_proto_max) {
                info->range.max_proto.all = info->range.min_proto.all;
        }
        return 0;
}
#endif

static const struct ovs_ct_len_tbl ovs_ct_attr_lens[OVS_CT_ATTR_MAX + 1] = {
        [OVS_CT_ATTR_COMMIT]        = { .minlen = 0, .maxlen = 0 },
        [OVS_CT_ATTR_FORCE_COMMIT]        = { .minlen = 0, .maxlen = 0 },
        [OVS_CT_ATTR_ZONE]        = { .minlen = sizeof(u16),
                                    .maxlen = sizeof(u16) },
        [OVS_CT_ATTR_MARK]        = { .minlen = sizeof(struct md_mark),
                                    .maxlen = sizeof(struct md_mark) },
        [OVS_CT_ATTR_LABELS]        = { .minlen = sizeof(struct md_labels),
                                    .maxlen = sizeof(struct md_labels) },
        [OVS_CT_ATTR_HELPER]        = { .minlen = 1,
                                    .maxlen = NF_CT_HELPER_NAME_LEN },
#if IS_ENABLED(CONFIG_NF_NAT)
        /* NAT length is checked when parsing the nested attributes. */
        [OVS_CT_ATTR_NAT]        = { .minlen = 0, .maxlen = INT_MAX },
#endif
        [OVS_CT_ATTR_EVENTMASK]        = { .minlen = sizeof(u32),
                                    .maxlen = sizeof(u32) },
        [OVS_CT_ATTR_TIMEOUT] = { .minlen = 1,
                                  .maxlen = CTNL_TIMEOUT_NAME_MAX },
};

static int parse_ct(const struct nlattr *attr, struct ovs_conntrack_info *info,
                    const char **helper, bool log)
{
        struct nlattr *a;
        int rem;

        nla_for_each_nested(a, attr, rem) {
                int type = nla_type(a);
                int maxlen;
                int minlen;

                if (type > OVS_CT_ATTR_MAX) {
                        OVS_NLERR(log,
                                  "Unknown conntrack attr (type=%d, max=%d)",
                                  type, OVS_CT_ATTR_MAX);
                        return -EINVAL;
                }

                maxlen = ovs_ct_attr_lens[type].maxlen;
                minlen = ovs_ct_attr_lens[type].minlen;
                if (nla_len(a) < minlen || nla_len(a) > maxlen) {
                        OVS_NLERR(log,
                                  "Conntrack attr type has unexpected length (type=%d, length=%d, expected=%d)",
                                  type, nla_len(a), maxlen);
                        return -EINVAL;
                }

                switch (type) {
                case OVS_CT_ATTR_FORCE_COMMIT:
                        info->force = true;
                        fallthrough;
                case OVS_CT_ATTR_COMMIT:
                        info->commit = true;
                        break;
#ifdef CONFIG_NF_CONNTRACK_ZONES
                case OVS_CT_ATTR_ZONE:
                        info->zone.id = nla_get_u16(a);
                        break;
#endif
#ifdef CONFIG_NF_CONNTRACK_MARK
                case OVS_CT_ATTR_MARK: {
                        struct md_mark *mark = nla_data(a);

                        if (!mark->mask) {
                                OVS_NLERR(log, "ct_mark mask cannot be 0");
                                return -EINVAL;
                        }
                        info->mark = *mark;
                        break;
                }
#endif
#ifdef CONFIG_NF_CONNTRACK_LABELS
                case OVS_CT_ATTR_LABELS: {
                        struct md_labels *labels = nla_data(a);

                        if (!labels_nonzero(&labels->mask)) {
                                OVS_NLERR(log, "ct_labels mask cannot be 0");
                                return -EINVAL;
                        }
                        info->labels = *labels;
                        break;
                }
#endif
                case OVS_CT_ATTR_HELPER:
                        *helper = nla_data(a);
                        if (!string_is_terminated(*helper, nla_len(a))) {
                                OVS_NLERR(log, "Invalid conntrack helper");
                                return -EINVAL;
                        }
                        break;
#if IS_ENABLED(CONFIG_NF_NAT)
                case OVS_CT_ATTR_NAT: {
                        int err = parse_nat(a, info, log);

                        if (err)
                                return err;
                        break;
                }
#endif
                case OVS_CT_ATTR_EVENTMASK:
                        info->have_eventmask = true;
                        info->eventmask = nla_get_u32(a);
                        break;
#ifdef CONFIG_NF_CONNTRACK_TIMEOUT
                case OVS_CT_ATTR_TIMEOUT:
                        memcpy(info->timeout, nla_data(a), nla_len(a));
                        if (!string_is_terminated(info->timeout, nla_len(a))) {
                                OVS_NLERR(log, "Invalid conntrack timeout");
                                return -EINVAL;
                        }
                        break;
#endif

                default:
                        OVS_NLERR(log, "Unknown conntrack attr (%d)",
                                  type);
                        return -EINVAL;
                }
        }

#ifdef CONFIG_NF_CONNTRACK_MARK
        if (!info->commit && info->mark.mask) {
                OVS_NLERR(log,
                          "Setting conntrack mark requires 'commit' flag.");
                return -EINVAL;
        }
#endif
#ifdef CONFIG_NF_CONNTRACK_LABELS
        if (!info->commit && labels_nonzero(&info->labels.mask)) {
                OVS_NLERR(log,
                          "Setting conntrack labels requires 'commit' flag.");
                return -EINVAL;
        }
#endif
        if (rem > 0) {
                OVS_NLERR(log, "Conntrack attr has %d unknown bytes", rem);
                return -EINVAL;
        }

        return 0;
}

bool ovs_ct_verify(struct net *net, enum ovs_key_attr attr)
{
        if (attr == OVS_KEY_ATTR_CT_STATE)
                return true;
        if (IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES) &&
            attr == OVS_KEY_ATTR_CT_ZONE)
                return true;
        if (IS_ENABLED(CONFIG_NF_CONNTRACK_MARK) &&
            attr == OVS_KEY_ATTR_CT_MARK)
                return true;
        if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS) &&
            attr == OVS_KEY_ATTR_CT_LABELS) {
                struct ovs_net *ovs_net = net_generic(net, ovs_net_id);

                return ovs_net->xt_label;
        }

        return false;
}

int ovs_ct_copy_action(struct net *net, const struct nlattr *attr,
                       const struct sw_flow_key *key,
                       struct sw_flow_actions **sfa,  bool log)
{
        struct ovs_conntrack_info ct_info;
        const char *helper = NULL;
        u16 family;
        int err;

        family = key_to_nfproto(key);
        if (family == NFPROTO_UNSPEC) {
                OVS_NLERR(log, "ct family unspecified");
                return -EINVAL;
        }

        memset(&ct_info, 0, sizeof(ct_info));
        ct_info.family = family;

        nf_ct_zone_init(&ct_info.zone, NF_CT_DEFAULT_ZONE_ID,
                        NF_CT_DEFAULT_ZONE_DIR, 0);

        err = parse_ct(attr, &ct_info, &helper, log);
        if (err)
                return err;

        /* Set up template for tracking connections in specific zones. */
        ct_info.ct = nf_ct_tmpl_alloc(net, &ct_info.zone, GFP_KERNEL);
        if (!ct_info.ct) {
                OVS_NLERR(log, "Failed to allocate conntrack template");
                return -ENOMEM;
        }

        if (ct_info.timeout[0]) {
                if (nf_ct_set_timeout(net, ct_info.ct, family, key->ip.proto,
                                      ct_info.timeout))
                        OVS_NLERR(log,
                                  "Failed to associated timeout policy '%s'",
                                  ct_info.timeout);
                else
                        ct_info.nf_ct_timeout = rcu_dereference(
                                nf_ct_timeout_find(ct_info.ct)->timeout);

        }

        if (helper) {
                err = nf_ct_add_helper(ct_info.ct, helper, ct_info.family,
                                       key->ip.proto, ct_info.nat, &ct_info.helper);
                if (err) {
                        OVS_NLERR(log, "Failed to add %s helper %d", helper, err);
                        goto err_free_ct;
                }
        }

        err = ovs_nla_add_action(sfa, OVS_ACTION_ATTR_CT, &ct_info,
                                 sizeof(ct_info), log);
        if (err)
                goto err_free_ct;

        if (ct_info.commit)
                __set_bit(IPS_CONFIRMED_BIT, &ct_info.ct->status);
        return 0;
err_free_ct:
        __ovs_ct_free_action(&ct_info);
        return err;
}

#if IS_ENABLED(CONFIG_NF_NAT)
static bool ovs_ct_nat_to_attr(const struct ovs_conntrack_info *info,
                               struct sk_buff *skb)
{
        struct nlattr *start;

        start = nla_nest_start_noflag(skb, OVS_CT_ATTR_NAT);
        if (!start)
                return false;

        if (info->nat & OVS_CT_SRC_NAT) {
                if (nla_put_flag(skb, OVS_NAT_ATTR_SRC))
                        return false;
        } else if (info->nat & OVS_CT_DST_NAT) {
                if (nla_put_flag(skb, OVS_NAT_ATTR_DST))
                        return false;
        } else {
                goto out;
        }

        if (info->range.flags & NF_NAT_RANGE_MAP_IPS) {
                if (IS_ENABLED(CONFIG_NF_NAT) &&
                    info->family == NFPROTO_IPV4) {
                        if (nla_put_in_addr(skb, OVS_NAT_ATTR_IP_MIN,
                                            info->range.min_addr.ip) ||
                            (info->range.max_addr.ip
                             != info->range.min_addr.ip &&
                             (nla_put_in_addr(skb, OVS_NAT_ATTR_IP_MAX,
                                              info->range.max_addr.ip))))
                                return false;
                } else if (IS_ENABLED(CONFIG_IPV6) &&
                           info->family == NFPROTO_IPV6) {
                        if (nla_put_in6_addr(skb, OVS_NAT_ATTR_IP_MIN,
                                             &info->range.min_addr.in6) ||
                            (memcmp(&info->range.max_addr.in6,
                                    &info->range.min_addr.in6,
                                    sizeof(info->range.max_addr.in6)) &&
                             (nla_put_in6_addr(skb, OVS_NAT_ATTR_IP_MAX,
                                               &info->range.max_addr.in6))))
                                return false;
                } else {
                        return false;
                }
        }
        if (info->range.flags & NF_NAT_RANGE_PROTO_SPECIFIED &&
            (nla_put_u16(skb, OVS_NAT_ATTR_PROTO_MIN,
                         ntohs(info->range.min_proto.all)) ||
             (info->range.max_proto.all != info->range.min_proto.all &&
              nla_put_u16(skb, OVS_NAT_ATTR_PROTO_MAX,
                          ntohs(info->range.max_proto.all)))))
                return false;

        if (info->range.flags & NF_NAT_RANGE_PERSISTENT &&
            nla_put_flag(skb, OVS_NAT_ATTR_PERSISTENT))
                return false;
        if (info->range.flags & NF_NAT_RANGE_PROTO_RANDOM &&
            nla_put_flag(skb, OVS_NAT_ATTR_PROTO_HASH))
                return false;
        if (info->range.flags & NF_NAT_RANGE_PROTO_RANDOM_FULLY &&
            nla_put_flag(skb, OVS_NAT_ATTR_PROTO_RANDOM))
                return false;
out:
        nla_nest_end(skb, start);

        return true;
}
#endif

int ovs_ct_action_to_attr(const struct ovs_conntrack_info *ct_info,
                          struct sk_buff *skb)
{
        struct nlattr *start;

        start = nla_nest_start_noflag(skb, OVS_ACTION_ATTR_CT);
        if (!start)
                return -EMSGSIZE;

        if (ct_info->commit && nla_put_flag(skb, ct_info->force
                                            ? OVS_CT_ATTR_FORCE_COMMIT
                                            : OVS_CT_ATTR_COMMIT))
                return -EMSGSIZE;
        if (IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES) &&
            nla_put_u16(skb, OVS_CT_ATTR_ZONE, ct_info->zone.id))
                return -EMSGSIZE;
        if (IS_ENABLED(CONFIG_NF_CONNTRACK_MARK) && ct_info->mark.mask &&
            nla_put(skb, OVS_CT_ATTR_MARK, sizeof(ct_info->mark),
                    &ct_info->mark))
                return -EMSGSIZE;
        if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS) &&
            labels_nonzero(&ct_info->labels.mask) &&
            nla_put(skb, OVS_CT_ATTR_LABELS, sizeof(ct_info->labels),
                    &ct_info->labels))
                return -EMSGSIZE;
        if (ct_info->helper) {
                if (nla_put_string(skb, OVS_CT_ATTR_HELPER,
                                   ct_info->helper->name))
                        return -EMSGSIZE;
        }
        if (ct_info->have_eventmask &&
            nla_put_u32(skb, OVS_CT_ATTR_EVENTMASK, ct_info->eventmask))
                return -EMSGSIZE;
        if (ct_info->timeout[0]) {
                if (nla_put_string(skb, OVS_CT_ATTR_TIMEOUT, ct_info->timeout))
                        return -EMSGSIZE;
        }

#if IS_ENABLED(CONFIG_NF_NAT)
        if (ct_info->nat && !ovs_ct_nat_to_attr(ct_info, skb))
                return -EMSGSIZE;
#endif
        nla_nest_end(skb, start);

        return 0;
}

void ovs_ct_free_action(const struct nlattr *a)
{
        struct ovs_conntrack_info *ct_info = nla_data(a);

        __ovs_ct_free_action(ct_info);
}

static void __ovs_ct_free_action(struct ovs_conntrack_info *ct_info)
{
        if (ct_info->helper) {
#if IS_ENABLED(CONFIG_NF_NAT)
                if (ct_info->nat)
                        nf_nat_helper_put(ct_info->helper);
#endif
                nf_conntrack_helper_put(ct_info->helper);
        }
        if (ct_info->ct) {
                if (ct_info->timeout[0])
                        nf_ct_destroy_timeout(ct_info->ct);
                nf_ct_tmpl_free(ct_info->ct);
        }
}

#if        IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT)
static int ovs_ct_limit_init(struct net *net, struct ovs_net *ovs_net)
{
        int i, err;

        ovs_net->ct_limit_info = kmalloc(sizeof(*ovs_net->ct_limit_info),
                                         GFP_KERNEL);
        if (!ovs_net->ct_limit_info)
                return -ENOMEM;

        ovs_net->ct_limit_info->default_limit = OVS_CT_LIMIT_DEFAULT;
        ovs_net->ct_limit_info->limits =
                kmalloc_array(CT_LIMIT_HASH_BUCKETS, sizeof(struct hlist_head),
                              GFP_KERNEL);
        if (!ovs_net->ct_limit_info->limits) {
                kfree(ovs_net->ct_limit_info);
                return -ENOMEM;
        }

        for (i = 0; i < CT_LIMIT_HASH_BUCKETS; i++)
                INIT_HLIST_HEAD(&ovs_net->ct_limit_info->limits[i]);

        ovs_net->ct_limit_info->data = nf_conncount_init(net, sizeof(u32));

        if (IS_ERR(ovs_net->ct_limit_info->data)) {
                err = PTR_ERR(ovs_net->ct_limit_info->data);
                kfree(ovs_net->ct_limit_info->limits);
                kfree(ovs_net->ct_limit_info);
                pr_err("openvswitch: failed to init nf_conncount %d\n", err);
                return err;
        }
        return 0;
}

static void ovs_ct_limit_exit(struct net *net, struct ovs_net *ovs_net)
{
        const struct ovs_ct_limit_info *info = ovs_net->ct_limit_info;
        int i;

        nf_conncount_destroy(net, info->data);
        for (i = 0; i < CT_LIMIT_HASH_BUCKETS; ++i) {
                struct hlist_head *head = &info->limits[i];
                struct ovs_ct_limit *ct_limit;
                struct hlist_node *next;

                hlist_for_each_entry_safe(ct_limit, next, head, hlist_node)
                        kfree_rcu(ct_limit, rcu);
        }
        kfree(info->limits);
        kfree(info);
}

static struct sk_buff *
ovs_ct_limit_cmd_reply_start(struct genl_info *info, u8 cmd,
                             struct ovs_header **ovs_reply_header)
{
        struct ovs_header *ovs_header = genl_info_userhdr(info);
        struct sk_buff *skb;

        skb = genlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!skb)
                return ERR_PTR(-ENOMEM);

        *ovs_reply_header = genlmsg_put(skb, info->snd_portid,
                                        info->snd_seq,
                                        &dp_ct_limit_genl_family, 0, cmd);

        if (!*ovs_reply_header) {
                nlmsg_free(skb);
                return ERR_PTR(-EMSGSIZE);
        }
        (*ovs_reply_header)->dp_ifindex = ovs_header->dp_ifindex;

        return skb;
}

static bool check_zone_id(int zone_id, u16 *pzone)
{
        if (zone_id >= 0 && zone_id <= 65535) {
                *pzone = (u16)zone_id;
                return true;
        }
        return false;
}

static int ovs_ct_limit_set_zone_limit(struct nlattr *nla_zone_limit,
                                       struct ovs_ct_limit_info *info)
{
        struct ovs_zone_limit *zone_limit;
        int rem;
        u16 zone;

        rem = NLA_ALIGN(nla_len(nla_zone_limit));
        zone_limit = (struct ovs_zone_limit *)nla_data(nla_zone_limit);

        while (rem >= sizeof(*zone_limit)) {
                if (unlikely(zone_limit->zone_id ==
                                OVS_ZONE_LIMIT_DEFAULT_ZONE)) {
                        ovs_lock();
                        info->default_limit = zone_limit->limit;
                        ovs_unlock();
                } else if (unlikely(!check_zone_id(
                                zone_limit->zone_id, &zone))) {
                        OVS_NLERR(true, "zone id is out of range");
                } else {
                        struct ovs_ct_limit *ct_limit;

                        ct_limit = kmalloc(sizeof(*ct_limit),
                                           GFP_KERNEL_ACCOUNT);
                        if (!ct_limit)
                                return -ENOMEM;

                        ct_limit->zone = zone;
                        ct_limit->limit = zone_limit->limit;

                        ovs_lock();
                        ct_limit_set(info, ct_limit);
                        ovs_unlock();
                }
                rem -= NLA_ALIGN(sizeof(*zone_limit));
                zone_limit = (struct ovs_zone_limit *)((u8 *)zone_limit +
                                NLA_ALIGN(sizeof(*zone_limit)));
        }

        if (rem)
                OVS_NLERR(true, "set zone limit has %d unknown bytes", rem);

        return 0;
}

static int ovs_ct_limit_del_zone_limit(struct nlattr *nla_zone_limit,
                                       struct ovs_ct_limit_info *info)
{
        struct ovs_zone_limit *zone_limit;
        int rem;
        u16 zone;

        rem = NLA_ALIGN(nla_len(nla_zone_limit));
        zone_limit = (struct ovs_zone_limit *)nla_data(nla_zone_limit);

        while (rem >= sizeof(*zone_limit)) {
                if (unlikely(zone_limit->zone_id ==
                                OVS_ZONE_LIMIT_DEFAULT_ZONE)) {
                        ovs_lock();
                        info->default_limit = OVS_CT_LIMIT_DEFAULT;
                        ovs_unlock();
                } else if (unlikely(!check_zone_id(
                                zone_limit->zone_id, &zone))) {
                        OVS_NLERR(true, "zone id is out of range");
                } else {
                        ovs_lock();
                        ct_limit_del(info, zone);
                        ovs_unlock();
                }
                rem -= NLA_ALIGN(sizeof(*zone_limit));
                zone_limit = (struct ovs_zone_limit *)((u8 *)zone_limit +
                                NLA_ALIGN(sizeof(*zone_limit)));
        }

        if (rem)
                OVS_NLERR(true, "del zone limit has %d unknown bytes", rem);

        return 0;
}

static int ovs_ct_limit_get_default_limit(struct ovs_ct_limit_info *info,
                                          struct sk_buff *reply)
{
        struct ovs_zone_limit zone_limit = {
                .zone_id = OVS_ZONE_LIMIT_DEFAULT_ZONE,
                .limit   = info->default_limit,
        };

        return nla_put_nohdr(reply, sizeof(zone_limit), &zone_limit);
}

static int __ovs_ct_limit_get_zone_limit(struct net *net,
                                         struct nf_conncount_data *data,
                                         u16 zone_id, u32 limit,
                                         struct sk_buff *reply)
{
        struct nf_conntrack_zone ct_zone;
        struct ovs_zone_limit zone_limit;
        u32 conncount_key = zone_id;

        zone_limit.zone_id = zone_id;
        zone_limit.limit = limit;
        nf_ct_zone_init(&ct_zone, zone_id, NF_CT_DEFAULT_ZONE_DIR, 0);

        zone_limit.count = nf_conncount_count(net, data, &conncount_key, NULL,
                                              &ct_zone);
        return nla_put_nohdr(reply, sizeof(zone_limit), &zone_limit);
}

static int ovs_ct_limit_get_zone_limit(struct net *net,
                                       struct nlattr *nla_zone_limit,
                                       struct ovs_ct_limit_info *info,
                                       struct sk_buff *reply)
{
        struct ovs_zone_limit *zone_limit;
        int rem, err;
        u32 limit;
        u16 zone;

        rem = NLA_ALIGN(nla_len(nla_zone_limit));
        zone_limit = (struct ovs_zone_limit *)nla_data(nla_zone_limit);

        while (rem >= sizeof(*zone_limit)) {
                if (unlikely(zone_limit->zone_id ==
                                OVS_ZONE_LIMIT_DEFAULT_ZONE)) {
                        err = ovs_ct_limit_get_default_limit(info, reply);
                        if (err)
                                return err;
                } else if (unlikely(!check_zone_id(zone_limit->zone_id,
                                                        &zone))) {
                        OVS_NLERR(true, "zone id is out of range");
                } else {
                        rcu_read_lock();
                        limit = ct_limit_get(info, zone);
                        rcu_read_unlock();

                        err = __ovs_ct_limit_get_zone_limit(
                                net, info->data, zone, limit, reply);
                        if (err)
                                return err;
                }
                rem -= NLA_ALIGN(sizeof(*zone_limit));
                zone_limit = (struct ovs_zone_limit *)((u8 *)zone_limit +
                                NLA_ALIGN(sizeof(*zone_limit)));
        }

        if (rem)
                OVS_NLERR(true, "get zone limit has %d unknown bytes", rem);

        return 0;
}

static int ovs_ct_limit_get_all_zone_limit(struct net *net,
                                           struct ovs_ct_limit_info *info,
                                           struct sk_buff *reply)
{
        struct ovs_ct_limit *ct_limit;
        struct hlist_head *head;
        int i, err = 0;

        err = ovs_ct_limit_get_default_limit(info, reply);
        if (err)
                return err;

        rcu_read_lock();
        for (i = 0; i < CT_LIMIT_HASH_BUCKETS; ++i) {
                head = &info->limits[i];
                hlist_for_each_entry_rcu(ct_limit, head, hlist_node) {
                        err = __ovs_ct_limit_get_zone_limit(net, info->data,
                                ct_limit->zone, ct_limit->limit, reply);
                        if (err)
                                goto exit_err;
                }
        }

exit_err:
        rcu_read_unlock();
        return err;
}

static int ovs_ct_limit_cmd_set(struct sk_buff *skb, struct genl_info *info)
{
        struct nlattr **a = info->attrs;
        struct sk_buff *reply;
        struct ovs_header *ovs_reply_header;
        struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id);
        struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info;
        int err;

        reply = ovs_ct_limit_cmd_reply_start(info, OVS_CT_LIMIT_CMD_SET,
                                             &ovs_reply_header);
        if (IS_ERR(reply))
                return PTR_ERR(reply);

        if (!a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT]) {
                err = -EINVAL;
                goto exit_err;
        }

        err = ovs_ct_limit_set_zone_limit(a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT],
                                          ct_limit_info);
        if (err)
                goto exit_err;

        static_branch_enable(&ovs_ct_limit_enabled);

        genlmsg_end(reply, ovs_reply_header);
        return genlmsg_reply(reply, info);

exit_err:
        nlmsg_free(reply);
        return err;
}

static int ovs_ct_limit_cmd_del(struct sk_buff *skb, struct genl_info *info)
{
        struct nlattr **a = info->attrs;
        struct sk_buff *reply;
        struct ovs_header *ovs_reply_header;
        struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id);
        struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info;
        int err;

        reply = ovs_ct_limit_cmd_reply_start(info, OVS_CT_LIMIT_CMD_DEL,
                                             &ovs_reply_header);
        if (IS_ERR(reply))
                return PTR_ERR(reply);

        if (!a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT]) {
                err = -EINVAL;
                goto exit_err;
        }

        err = ovs_ct_limit_del_zone_limit(a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT],
                                          ct_limit_info);
        if (err)
                goto exit_err;

        genlmsg_end(reply, ovs_reply_header);
        return genlmsg_reply(reply, info);

exit_err:
        nlmsg_free(reply);
        return err;
}

static int ovs_ct_limit_cmd_get(struct sk_buff *skb, struct genl_info *info)
{
        struct nlattr **a = info->attrs;
        struct nlattr *nla_reply;
        struct sk_buff *reply;
        struct ovs_header *ovs_reply_header;
        struct net *net = sock_net(skb->sk);
        struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
        struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info;
        int err;

        reply = ovs_ct_limit_cmd_reply_start(info, OVS_CT_LIMIT_CMD_GET,
                                             &ovs_reply_header);
        if (IS_ERR(reply))
                return PTR_ERR(reply);

        nla_reply = nla_nest_start_noflag(reply, OVS_CT_LIMIT_ATTR_ZONE_LIMIT);
        if (!nla_reply) {
                err = -EMSGSIZE;
                goto exit_err;
        }

        if (a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT]) {
                err = ovs_ct_limit_get_zone_limit(
                        net, a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT], ct_limit_info,
                        reply);
                if (err)
                        goto exit_err;
        } else {
                err = ovs_ct_limit_get_all_zone_limit(net, ct_limit_info,
                                                      reply);
                if (err)
                        goto exit_err;
        }

        nla_nest_end(reply, nla_reply);
        genlmsg_end(reply, ovs_reply_header);
        return genlmsg_reply(reply, info);

exit_err:
        nlmsg_free(reply);
        return err;
}

static const struct genl_small_ops ct_limit_genl_ops[] = {
        { .cmd = OVS_CT_LIMIT_CMD_SET,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN
                                               * privilege.
                                               */
                .doit = ovs_ct_limit_cmd_set,
        },
        { .cmd = OVS_CT_LIMIT_CMD_DEL,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN
                                               * privilege.
                                               */
                .doit = ovs_ct_limit_cmd_del,
        },
        { .cmd = OVS_CT_LIMIT_CMD_GET,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = 0,                  /* OK for unprivileged users. */
                .doit = ovs_ct_limit_cmd_get,
        },
};

static const struct genl_multicast_group ovs_ct_limit_multicast_group = {
        .name = OVS_CT_LIMIT_MCGROUP,
};

struct genl_family dp_ct_limit_genl_family __ro_after_init = {
        .hdrsize = sizeof(struct ovs_header),
        .name = OVS_CT_LIMIT_FAMILY,
        .version = OVS_CT_LIMIT_VERSION,
        .maxattr = OVS_CT_LIMIT_ATTR_MAX,
        .policy = ct_limit_policy,
        .netnsok = true,
        .parallel_ops = true,
        .small_ops = ct_limit_genl_ops,
        .n_small_ops = ARRAY_SIZE(ct_limit_genl_ops),
        .resv_start_op = OVS_CT_LIMIT_CMD_GET + 1,
        .mcgrps = &ovs_ct_limit_multicast_group,
        .n_mcgrps = 1,
        .module = THIS_MODULE,
};
#endif

int ovs_ct_init(struct net *net)
{
        unsigned int n_bits = sizeof(struct ovs_key_ct_labels) * BITS_PER_BYTE;
        struct ovs_net *ovs_net = net_generic(net, ovs_net_id);

        if (nf_connlabels_get(net, n_bits - 1)) {
                ovs_net->xt_label = false;
                OVS_NLERR(true, "Failed to set connlabel length");
        } else {
                ovs_net->xt_label = true;
        }

#if        IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT)
        return ovs_ct_limit_init(net, ovs_net);
#else
        return 0;
#endif
}

void ovs_ct_exit(struct net *net)
{
        struct ovs_net *ovs_net = net_generic(net, ovs_net_id);

#if        IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT)
        ovs_ct_limit_exit(net, ovs_net);
#endif

        if (ovs_net->xt_label)
                nf_connlabels_put(net);
}




























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   29 

































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
// SPDX-License-Identifier: BSD-3-Clause
/*
 * linux/net/sunrpc/auth_gss/auth_gss.c
 *
 * RPCSEC_GSS client authentication.
 *
 *  Copyright (c) 2000 The Regents of the University of Michigan.
 *  All rights reserved.
 *
 *  Dug Song       <dugsong@monkey.org>
 *  Andy Adamson   <andros@umich.edu>
 */

#include <linux/module.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/pagemap.h>
#include <linux/sunrpc/clnt.h>
#include <linux/sunrpc/auth.h>
#include <linux/sunrpc/auth_gss.h>
#include <linux/sunrpc/gss_krb5.h>
#include <linux/sunrpc/svcauth_gss.h>
#include <linux/sunrpc/gss_err.h>
#include <linux/workqueue.h>
#include <linux/sunrpc/rpc_pipe_fs.h>
#include <linux/sunrpc/gss_api.h>
#include <linux/uaccess.h>
#include <linux/hashtable.h>

#include "auth_gss_internal.h"
#include "../netns.h"

#include <trace/events/rpcgss.h>

static const struct rpc_authops authgss_ops;

static const struct rpc_credops gss_credops;
static const struct rpc_credops gss_nullops;

#define GSS_RETRY_EXPIRED 5
static unsigned int gss_expired_cred_retry_delay = GSS_RETRY_EXPIRED;

#define GSS_KEY_EXPIRE_TIMEO 240
static unsigned int gss_key_expire_timeo = GSS_KEY_EXPIRE_TIMEO;

#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
# define RPCDBG_FACILITY        RPCDBG_AUTH
#endif

/*
 * This compile-time check verifies that we will not exceed the
 * slack space allotted by the client and server auth_gss code
 * before they call gss_wrap().
 */
#define GSS_KRB5_MAX_SLACK_NEEDED                                        \
        (GSS_KRB5_TOK_HDR_LEN                /* gss token header */                \
        + GSS_KRB5_MAX_CKSUM_LEN        /* gss token checksum */        \
        + GSS_KRB5_MAX_BLOCKSIZE        /* confounder */                \
        + GSS_KRB5_MAX_BLOCKSIZE        /* possible padding */                \
        + GSS_KRB5_TOK_HDR_LEN                /* encrypted hdr in v2 token */        \
        + GSS_KRB5_MAX_CKSUM_LEN        /* encryption hmac */                \
        + XDR_UNIT * 2                        /* RPC verifier */                \
        + GSS_KRB5_TOK_HDR_LEN                                                \
        + GSS_KRB5_MAX_CKSUM_LEN)

#define GSS_CRED_SLACK                (RPC_MAX_AUTH_SIZE * 2)
/* length of a krb5 verifier (48), plus data added before arguments when
 * using integrity (two 4-byte integers): */
#define GSS_VERF_SLACK                100

static DEFINE_HASHTABLE(gss_auth_hash_table, 4);
static DEFINE_SPINLOCK(gss_auth_hash_lock);

struct gss_pipe {
        struct rpc_pipe_dir_object pdo;
        struct rpc_pipe *pipe;
        struct rpc_clnt *clnt;
        const char *name;
        struct kref kref;
};

struct gss_auth {
        struct kref kref;
        struct hlist_node hash;
        struct rpc_auth rpc_auth;
        struct gss_api_mech *mech;
        enum rpc_gss_svc service;
        struct rpc_clnt *client;
        struct net        *net;
        netns_tracker        ns_tracker;
        /*
         * There are two upcall pipes; dentry[1], named "gssd", is used
         * for the new text-based upcall; dentry[0] is named after the
         * mechanism (for example, "krb5") and exists for
         * backwards-compatibility with older gssd's.
         */
        struct gss_pipe *gss_pipe[2];
        const char *target_name;
};

/* pipe_version >= 0 if and only if someone has a pipe open. */
static DEFINE_SPINLOCK(pipe_version_lock);
static struct rpc_wait_queue pipe_version_rpc_waitqueue;
static DECLARE_WAIT_QUEUE_HEAD(pipe_version_waitqueue);
static void gss_put_auth(struct gss_auth *gss_auth);

static void gss_free_ctx(struct gss_cl_ctx *);
static const struct rpc_pipe_ops gss_upcall_ops_v0;
static const struct rpc_pipe_ops gss_upcall_ops_v1;

static inline struct gss_cl_ctx *
gss_get_ctx(struct gss_cl_ctx *ctx)
{
        refcount_inc(&ctx->count);
        return ctx;
}

static inline void
gss_put_ctx(struct gss_cl_ctx *ctx)
{
        if (refcount_dec_and_test(&ctx->count))
                gss_free_ctx(ctx);
}

/* gss_cred_set_ctx:
 * called by gss_upcall_callback and gss_create_upcall in order
 * to set the gss context. The actual exchange of an old context
 * and a new one is protected by the pipe->lock.
 */
static void
gss_cred_set_ctx(struct rpc_cred *cred, struct gss_cl_ctx *ctx)
{
        struct gss_cred *gss_cred = container_of(cred, struct gss_cred, gc_base);

        if (!test_bit(RPCAUTH_CRED_NEW, &cred->cr_flags))
                return;
        gss_get_ctx(ctx);
        rcu_assign_pointer(gss_cred->gc_ctx, ctx);
        set_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags);
        smp_mb__before_atomic();
        clear_bit(RPCAUTH_CRED_NEW, &cred->cr_flags);
}

static struct gss_cl_ctx *
gss_cred_get_ctx(struct rpc_cred *cred)
{
        struct gss_cred *gss_cred = container_of(cred, struct gss_cred, gc_base);
        struct gss_cl_ctx *ctx = NULL;

        rcu_read_lock();
        ctx = rcu_dereference(gss_cred->gc_ctx);
        if (ctx)
                gss_get_ctx(ctx);
        rcu_read_unlock();
        return ctx;
}

static struct gss_cl_ctx *
gss_alloc_context(void)
{
        struct gss_cl_ctx *ctx;

        ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
        if (ctx != NULL) {
                ctx->gc_proc = RPC_GSS_PROC_DATA;
                ctx->gc_seq = 1;        /* NetApp 6.4R1 doesn't accept seq. no. 0 */
                spin_lock_init(&ctx->gc_seq_lock);
                refcount_set(&ctx->count,1);
        }
        return ctx;
}

#define GSSD_MIN_TIMEOUT (60 * 60)
static const void *
gss_fill_context(const void *p, const void *end, struct gss_cl_ctx *ctx, struct gss_api_mech *gm)
{
        const void *q;
        unsigned int seclen;
        unsigned int timeout;
        unsigned long now = jiffies;
        u32 window_size;
        int ret;

        /* First unsigned int gives the remaining lifetime in seconds of the
         * credential - e.g. the remaining TGT lifetime for Kerberos or
         * the -t value passed to GSSD.
         */
        p = simple_get_bytes(p, end, &timeout, sizeof(timeout));
        if (IS_ERR(p))
                goto err;
        if (timeout == 0)
                timeout = GSSD_MIN_TIMEOUT;
        ctx->gc_expiry = now + ((unsigned long)timeout * HZ);
        /* Sequence number window. Determines the maximum number of
         * simultaneous requests
         */
        p = simple_get_bytes(p, end, &window_size, sizeof(window_size));
        if (IS_ERR(p))
                goto err;
        ctx->gc_win = window_size;
        /* gssd signals an error by passing ctx->gc_win = 0: */
        if (ctx->gc_win == 0) {
                /*
                 * in which case, p points to an error code. Anything other
                 * than -EKEYEXPIRED gets converted to -EACCES.
                 */
                p = simple_get_bytes(p, end, &ret, sizeof(ret));
                if (!IS_ERR(p))
                        p = (ret == -EKEYEXPIRED) ? ERR_PTR(-EKEYEXPIRED) :
                                                    ERR_PTR(-EACCES);
                goto err;
        }
        /* copy the opaque wire context */
        p = simple_get_netobj(p, end, &ctx->gc_wire_ctx);
        if (IS_ERR(p))
                goto err;
        /* import the opaque security context */
        p  = simple_get_bytes(p, end, &seclen, sizeof(seclen));
        if (IS_ERR(p))
                goto err;
        q = (const void *)((const char *)p + seclen);
        if (unlikely(q > end || q < p)) {
                p = ERR_PTR(-EFAULT);
                goto err;
        }
        ret = gss_import_sec_context(p, seclen, gm, &ctx->gc_gss_ctx, NULL, GFP_KERNEL);
        if (ret < 0) {
                trace_rpcgss_import_ctx(ret);
                p = ERR_PTR(ret);
                goto err;
        }

        /* is there any trailing data? */
        if (q == end) {
                p = q;
                goto done;
        }

        /* pull in acceptor name (if there is one) */
        p = simple_get_netobj(q, end, &ctx->gc_acceptor);
        if (IS_ERR(p))
                goto err;
done:
        trace_rpcgss_context(window_size, ctx->gc_expiry, now, timeout,
                             ctx->gc_acceptor.len, ctx->gc_acceptor.data);
err:
        return p;
}

/* XXX: Need some documentation about why UPCALL_BUF_LEN is so small.
 *        Is user space expecting no more than UPCALL_BUF_LEN bytes?
 *        Note that there are now _two_ NI_MAXHOST sized data items
 *        being passed in this string.
 */
#define UPCALL_BUF_LEN        256

struct gss_upcall_msg {
        refcount_t count;
        kuid_t        uid;
        const char *service_name;
        struct rpc_pipe_msg msg;
        struct list_head list;
        struct gss_auth *auth;
        struct rpc_pipe *pipe;
        struct rpc_wait_queue rpc_waitqueue;
        wait_queue_head_t waitqueue;
        struct gss_cl_ctx *ctx;
        char databuf[UPCALL_BUF_LEN];
};

static int get_pipe_version(struct net *net)
{
        struct sunrpc_net *sn = net_generic(net, sunrpc_net_id);
        int ret;

        spin_lock(&pipe_version_lock);
        if (sn->pipe_version >= 0) {
                atomic_inc(&sn->pipe_users);
                ret = sn->pipe_version;
        } else
                ret = -EAGAIN;
        spin_unlock(&pipe_version_lock);
        return ret;
}

static void put_pipe_version(struct net *net)
{
        struct sunrpc_net *sn = net_generic(net, sunrpc_net_id);

        if (atomic_dec_and_lock(&sn->pipe_users, &pipe_version_lock)) {
                sn->pipe_version = -1;
                spin_unlock(&pipe_version_lock);
        }
}

static void
gss_release_msg(struct gss_upcall_msg *gss_msg)
{
        struct net *net = gss_msg->auth->net;
        if (!refcount_dec_and_test(&gss_msg->count))
                return;
        put_pipe_version(net);
        BUG_ON(!list_empty(&gss_msg->list));
        if (gss_msg->ctx != NULL)
                gss_put_ctx(gss_msg->ctx);
        rpc_destroy_wait_queue(&gss_msg->rpc_waitqueue);
        gss_put_auth(gss_msg->auth);
        kfree_const(gss_msg->service_name);
        kfree(gss_msg);
}

static struct gss_upcall_msg *
__gss_find_upcall(struct rpc_pipe *pipe, kuid_t uid, const struct gss_auth *auth)
{
        struct gss_upcall_msg *pos;
        list_for_each_entry(pos, &pipe->in_downcall, list) {
                if (!uid_eq(pos->uid, uid))
                        continue;
                if (pos->auth->service != auth->service)
                        continue;
                refcount_inc(&pos->count);
                return pos;
        }
        return NULL;
}

/* Try to add an upcall to the pipefs queue.
 * If an upcall owned by our uid already exists, then we return a reference
 * to that upcall instead of adding the new upcall.
 */
static inline struct gss_upcall_msg *
gss_add_msg(struct gss_upcall_msg *gss_msg)
{
        struct rpc_pipe *pipe = gss_msg->pipe;
        struct gss_upcall_msg *old;

        spin_lock(&pipe->lock);
        old = __gss_find_upcall(pipe, gss_msg->uid, gss_msg->auth);
        if (old == NULL) {
                refcount_inc(&gss_msg->count);
                list_add(&gss_msg->list, &pipe->in_downcall);
        } else
                gss_msg = old;
        spin_unlock(&pipe->lock);
        return gss_msg;
}

static void
__gss_unhash_msg(struct gss_upcall_msg *gss_msg)
{
        list_del_init(&gss_msg->list);
        rpc_wake_up_status(&gss_msg->rpc_waitqueue, gss_msg->msg.errno);
        wake_up_all(&gss_msg->waitqueue);
        refcount_dec(&gss_msg->count);
}

static void
gss_unhash_msg(struct gss_upcall_msg *gss_msg)
{
        struct rpc_pipe *pipe = gss_msg->pipe;

        if (list_empty(&gss_msg->list))
                return;
        spin_lock(&pipe->lock);
        if (!list_empty(&gss_msg->list))
                __gss_unhash_msg(gss_msg);
        spin_unlock(&pipe->lock);
}

static void
gss_handle_downcall_result(struct gss_cred *gss_cred, struct gss_upcall_msg *gss_msg)
{
        switch (gss_msg->msg.errno) {
        case 0:
                if (gss_msg->ctx == NULL)
                        break;
                clear_bit(RPCAUTH_CRED_NEGATIVE, &gss_cred->gc_base.cr_flags);
                gss_cred_set_ctx(&gss_cred->gc_base, gss_msg->ctx);
                break;
        case -EKEYEXPIRED:
                set_bit(RPCAUTH_CRED_NEGATIVE, &gss_cred->gc_base.cr_flags);
        }
        gss_cred->gc_upcall_timestamp = jiffies;
        gss_cred->gc_upcall = NULL;
        rpc_wake_up_status(&gss_msg->rpc_waitqueue, gss_msg->msg.errno);
}

static void
gss_upcall_callback(struct rpc_task *task)
{
        struct gss_cred *gss_cred = container_of(task->tk_rqstp->rq_cred,
                        struct gss_cred, gc_base);
        struct gss_upcall_msg *gss_msg = gss_cred->gc_upcall;
        struct rpc_pipe *pipe = gss_msg->pipe;

        spin_lock(&pipe->lock);
        gss_handle_downcall_result(gss_cred, gss_msg);
        spin_unlock(&pipe->lock);
        task->tk_status = gss_msg->msg.errno;
        gss_release_msg(gss_msg);
}

static void gss_encode_v0_msg(struct gss_upcall_msg *gss_msg,
                              const struct cred *cred)
{
        struct user_namespace *userns = cred->user_ns;

        uid_t uid = from_kuid_munged(userns, gss_msg->uid);
        memcpy(gss_msg->databuf, &uid, sizeof(uid));
        gss_msg->msg.data = gss_msg->databuf;
        gss_msg->msg.len = sizeof(uid);

        BUILD_BUG_ON(sizeof(uid) > sizeof(gss_msg->databuf));
}

static ssize_t
gss_v0_upcall(struct file *file, struct rpc_pipe_msg *msg,
                char __user *buf, size_t buflen)
{
        struct gss_upcall_msg *gss_msg = container_of(msg,
                                                      struct gss_upcall_msg,
                                                      msg);
        if (msg->copied == 0)
                gss_encode_v0_msg(gss_msg, file->f_cred);
        return rpc_pipe_generic_upcall(file, msg, buf, buflen);
}

static int gss_encode_v1_msg(struct gss_upcall_msg *gss_msg,
                                const char *service_name,
                                const char *target_name,
                                const struct cred *cred)
{
        struct user_namespace *userns = cred->user_ns;
        struct gss_api_mech *mech = gss_msg->auth->mech;
        char *p = gss_msg->databuf;
        size_t buflen = sizeof(gss_msg->databuf);
        int len;

        len = scnprintf(p, buflen, "mech=%s uid=%d", mech->gm_name,
                        from_kuid_munged(userns, gss_msg->uid));
        buflen -= len;
        p += len;
        gss_msg->msg.len = len;

        /*
         * target= is a full service principal that names the remote
         * identity that we are authenticating to.
         */
        if (target_name) {
                len = scnprintf(p, buflen, " target=%s", target_name);
                buflen -= len;
                p += len;
                gss_msg->msg.len += len;
        }

        /*
         * gssd uses service= and srchost= to select a matching key from
         * the system's keytab to use as the source principal.
         *
         * service= is the service name part of the source principal,
         * or "*" (meaning choose any).
         *
         * srchost= is the hostname part of the source principal. When
         * not provided, gssd uses the local hostname.
         */
        if (service_name) {
                char *c = strchr(service_name, '@');

                if (!c)
                        len = scnprintf(p, buflen, " service=%s",
                                        service_name);
                else
                        len = scnprintf(p, buflen,
                                        " service=%.*s srchost=%s",
                                        (int)(c - service_name),
                                        service_name, c + 1);
                buflen -= len;
                p += len;
                gss_msg->msg.len += len;
        }

        if (mech->gm_upcall_enctypes) {
                len = scnprintf(p, buflen, " enctypes=%s",
                                mech->gm_upcall_enctypes);
                buflen -= len;
                p += len;
                gss_msg->msg.len += len;
        }
        trace_rpcgss_upcall_msg(gss_msg->databuf);
        len = scnprintf(p, buflen, "\n");
        if (len == 0)
                goto out_overflow;
        gss_msg->msg.len += len;
        gss_msg->msg.data = gss_msg->databuf;
        return 0;
out_overflow:
        WARN_ON_ONCE(1);
        return -ENOMEM;
}

static ssize_t
gss_v1_upcall(struct file *file, struct rpc_pipe_msg *msg,
                char __user *buf, size_t buflen)
{
        struct gss_upcall_msg *gss_msg = container_of(msg,
                                                      struct gss_upcall_msg,
                                                      msg);
        int err;
        if (msg->copied == 0) {
                err = gss_encode_v1_msg(gss_msg,
                                        gss_msg->service_name,
                                        gss_msg->auth->target_name,
                                        file->f_cred);
                if (err)
                        return err;
        }
        return rpc_pipe_generic_upcall(file, msg, buf, buflen);
}

static struct gss_upcall_msg *
gss_alloc_msg(struct gss_auth *gss_auth,
                kuid_t uid, const char *service_name)
{
        struct gss_upcall_msg *gss_msg;
        int vers;
        int err = -ENOMEM;

        gss_msg = kzalloc(sizeof(*gss_msg), GFP_KERNEL);
        if (gss_msg == NULL)
                goto err;
        vers = get_pipe_version(gss_auth->net);
        err = vers;
        if (err < 0)
                goto err_free_msg;
        gss_msg->pipe = gss_auth->gss_pipe[vers]->pipe;
        INIT_LIST_HEAD(&gss_msg->list);
        rpc_init_wait_queue(&gss_msg->rpc_waitqueue, "RPCSEC_GSS upcall waitq");
        init_waitqueue_head(&gss_msg->waitqueue);
        refcount_set(&gss_msg->count, 1);
        gss_msg->uid = uid;
        gss_msg->auth = gss_auth;
        kref_get(&gss_auth->kref);
        if (service_name) {
                gss_msg->service_name = kstrdup_const(service_name, GFP_KERNEL);
                if (!gss_msg->service_name) {
                        err = -ENOMEM;
                        goto err_put_pipe_version;
                }
        }
        return gss_msg;
err_put_pipe_version:
        put_pipe_version(gss_auth->net);
err_free_msg:
        kfree(gss_msg);
err:
        return ERR_PTR(err);
}

static struct gss_upcall_msg *
gss_setup_upcall(struct gss_auth *gss_auth, struct rpc_cred *cred)
{
        struct gss_cred *gss_cred = container_of(cred,
                        struct gss_cred, gc_base);
        struct gss_upcall_msg *gss_new, *gss_msg;
        kuid_t uid = cred->cr_cred->fsuid;

        gss_new = gss_alloc_msg(gss_auth, uid, gss_cred->gc_principal);
        if (IS_ERR(gss_new))
                return gss_new;
        gss_msg = gss_add_msg(gss_new);
        if (gss_msg == gss_new) {
                int res;
                refcount_inc(&gss_msg->count);
                res = rpc_queue_upcall(gss_new->pipe, &gss_new->msg);
                if (res) {
                        gss_unhash_msg(gss_new);
                        refcount_dec(&gss_msg->count);
                        gss_release_msg(gss_new);
                        gss_msg = ERR_PTR(res);
                }
        } else
                gss_release_msg(gss_new);
        return gss_msg;
}

static void warn_gssd(void)
{
        dprintk("AUTH_GSS upcall failed. Please check user daemon is running.\n");
}

static inline int
gss_refresh_upcall(struct rpc_task *task)
{
        struct rpc_cred *cred = task->tk_rqstp->rq_cred;
        struct gss_auth *gss_auth = container_of(cred->cr_auth,
                        struct gss_auth, rpc_auth);
        struct gss_cred *gss_cred = container_of(cred,
                        struct gss_cred, gc_base);
        struct gss_upcall_msg *gss_msg;
        struct rpc_pipe *pipe;
        int err = 0;

        gss_msg = gss_setup_upcall(gss_auth, cred);
        if (PTR_ERR(gss_msg) == -EAGAIN) {
                /* XXX: warning on the first, under the assumption we
                 * shouldn't normally hit this case on a refresh. */
                warn_gssd();
                rpc_sleep_on_timeout(&pipe_version_rpc_waitqueue,
                                task, NULL, jiffies + (15 * HZ));
                err = -EAGAIN;
                goto out;
        }
        if (IS_ERR(gss_msg)) {
                err = PTR_ERR(gss_msg);
                goto out;
        }
        pipe = gss_msg->pipe;
        spin_lock(&pipe->lock);
        if (gss_cred->gc_upcall != NULL)
                rpc_sleep_on(&gss_cred->gc_upcall->rpc_waitqueue, task, NULL);
        else if (gss_msg->ctx == NULL && gss_msg->msg.errno >= 0) {
                gss_cred->gc_upcall = gss_msg;
                /* gss_upcall_callback will release the reference to gss_upcall_msg */
                refcount_inc(&gss_msg->count);
                rpc_sleep_on(&gss_msg->rpc_waitqueue, task, gss_upcall_callback);
        } else {
                gss_handle_downcall_result(gss_cred, gss_msg);
                err = gss_msg->msg.errno;
        }
        spin_unlock(&pipe->lock);
        gss_release_msg(gss_msg);
out:
        trace_rpcgss_upcall_result(from_kuid(&init_user_ns,
                                             cred->cr_cred->fsuid), err);
        return err;
}

static inline int
gss_create_upcall(struct gss_auth *gss_auth, struct gss_cred *gss_cred)
{
        struct net *net = gss_auth->net;
        struct sunrpc_net *sn = net_generic(net, sunrpc_net_id);
        struct rpc_pipe *pipe;
        struct rpc_cred *cred = &gss_cred->gc_base;
        struct gss_upcall_msg *gss_msg;
        DEFINE_WAIT(wait);
        int err;

retry:
        err = 0;
        /* if gssd is down, just skip upcalling altogether */
        if (!gssd_running(net)) {
                warn_gssd();
                err = -EACCES;
                goto out;
        }
        gss_msg = gss_setup_upcall(gss_auth, cred);
        if (PTR_ERR(gss_msg) == -EAGAIN) {
                err = wait_event_interruptible_timeout(pipe_version_waitqueue,
                                sn->pipe_version >= 0, 15 * HZ);
                if (sn->pipe_version < 0) {
                        warn_gssd();
                        err = -EACCES;
                }
                if (err < 0)
                        goto out;
                goto retry;
        }
        if (IS_ERR(gss_msg)) {
                err = PTR_ERR(gss_msg);
                goto out;
        }
        pipe = gss_msg->pipe;
        for (;;) {
                prepare_to_wait(&gss_msg->waitqueue, &wait, TASK_KILLABLE);
                spin_lock(&pipe->lock);
                if (gss_msg->ctx != NULL || gss_msg->msg.errno < 0) {
                        break;
                }
                spin_unlock(&pipe->lock);
                if (fatal_signal_pending(current)) {
                        err = -ERESTARTSYS;
                        goto out_intr;
                }
                schedule();
        }
        if (gss_msg->ctx) {
                trace_rpcgss_ctx_init(gss_cred);
                gss_cred_set_ctx(cred, gss_msg->ctx);
        } else {
                err = gss_msg->msg.errno;
        }
        spin_unlock(&pipe->lock);
out_intr:
        finish_wait(&gss_msg->waitqueue, &wait);
        gss_release_msg(gss_msg);
out:
        trace_rpcgss_upcall_result(from_kuid(&init_user_ns,
                                             cred->cr_cred->fsuid), err);
        return err;
}

static struct gss_upcall_msg *
gss_find_downcall(struct rpc_pipe *pipe, kuid_t uid)
{
        struct gss_upcall_msg *pos;
        list_for_each_entry(pos, &pipe->in_downcall, list) {
                if (!uid_eq(pos->uid, uid))
                        continue;
                if (!rpc_msg_is_inflight(&pos->msg))
                        continue;
                refcount_inc(&pos->count);
                return pos;
        }
        return NULL;
}

#define MSG_BUF_MAXSIZE 1024

static ssize_t
gss_pipe_downcall(struct file *filp, const char __user *src, size_t mlen)
{
        const void *p, *end;
        void *buf;
        struct gss_upcall_msg *gss_msg;
        struct rpc_pipe *pipe = RPC_I(file_inode(filp))->pipe;
        struct gss_cl_ctx *ctx;
        uid_t id;
        kuid_t uid;
        ssize_t err = -EFBIG;

        if (mlen > MSG_BUF_MAXSIZE)
                goto out;
        err = -ENOMEM;
        buf = kmalloc(mlen, GFP_KERNEL);
        if (!buf)
                goto out;

        err = -EFAULT;
        if (copy_from_user(buf, src, mlen))
                goto err;

        end = (const void *)((char *)buf + mlen);
        p = simple_get_bytes(buf, end, &id, sizeof(id));
        if (IS_ERR(p)) {
                err = PTR_ERR(p);
                goto err;
        }

        uid = make_kuid(current_user_ns(), id);
        if (!uid_valid(uid)) {
                err = -EINVAL;
                goto err;
        }

        err = -ENOMEM;
        ctx = gss_alloc_context();
        if (ctx == NULL)
                goto err;

        err = -ENOENT;
        /* Find a matching upcall */
        spin_lock(&pipe->lock);
        gss_msg = gss_find_downcall(pipe, uid);
        if (gss_msg == NULL) {
                spin_unlock(&pipe->lock);
                goto err_put_ctx;
        }
        list_del_init(&gss_msg->list);
        spin_unlock(&pipe->lock);

        p = gss_fill_context(p, end, ctx, gss_msg->auth->mech);
        if (IS_ERR(p)) {
                err = PTR_ERR(p);
                switch (err) {
                case -EACCES:
                case -EKEYEXPIRED:
                        gss_msg->msg.errno = err;
                        err = mlen;
                        break;
                case -EFAULT:
                case -ENOMEM:
                case -EINVAL:
                case -ENOSYS:
                        gss_msg->msg.errno = -EAGAIN;
                        break;
                default:
                        printk(KERN_CRIT "%s: bad return from "
                                "gss_fill_context: %zd\n", __func__, err);
                        gss_msg->msg.errno = -EIO;
                }
                goto err_release_msg;
        }
        gss_msg->ctx = gss_get_ctx(ctx);
        err = mlen;

err_release_msg:
        spin_lock(&pipe->lock);
        __gss_unhash_msg(gss_msg);
        spin_unlock(&pipe->lock);
        gss_release_msg(gss_msg);
err_put_ctx:
        gss_put_ctx(ctx);
err:
        kfree(buf);
out:
        return err;
}

static int gss_pipe_open(struct inode *inode, int new_version)
{
        struct net *net = inode->i_sb->s_fs_info;
        struct sunrpc_net *sn = net_generic(net, sunrpc_net_id);
        int ret = 0;

        spin_lock(&pipe_version_lock);
        if (sn->pipe_version < 0) {
                /* First open of any gss pipe determines the version: */
                sn->pipe_version = new_version;
                rpc_wake_up(&pipe_version_rpc_waitqueue);
                wake_up(&pipe_version_waitqueue);
        } else if (sn->pipe_version != new_version) {
                /* Trying to open a pipe of a different version */
                ret = -EBUSY;
                goto out;
        }
        atomic_inc(&sn->pipe_users);
out:
        spin_unlock(&pipe_version_lock);
        return ret;

}

static int gss_pipe_open_v0(struct inode *inode)
{
        return gss_pipe_open(inode, 0);
}

static int gss_pipe_open_v1(struct inode *inode)
{
        return gss_pipe_open(inode, 1);
}

static void
gss_pipe_release(struct inode *inode)
{
        struct net *net = inode->i_sb->s_fs_info;
        struct rpc_pipe *pipe = RPC_I(inode)->pipe;
        struct gss_upcall_msg *gss_msg;

restart:
        spin_lock(&pipe->lock);
        list_for_each_entry(gss_msg, &pipe->in_downcall, list) {

                if (!list_empty(&gss_msg->msg.list))
                        continue;
                gss_msg->msg.errno = -EPIPE;
                refcount_inc(&gss_msg->count);
                __gss_unhash_msg(gss_msg);
                spin_unlock(&pipe->lock);
                gss_release_msg(gss_msg);
                goto restart;
        }
        spin_unlock(&pipe->lock);

        put_pipe_version(net);
}

static void
gss_pipe_destroy_msg(struct rpc_pipe_msg *msg)
{
        struct gss_upcall_msg *gss_msg = container_of(msg, struct gss_upcall_msg, msg);

        if (msg->errno < 0) {
                refcount_inc(&gss_msg->count);
                gss_unhash_msg(gss_msg);
                if (msg->errno == -ETIMEDOUT)
                        warn_gssd();
                gss_release_msg(gss_msg);
        }
        gss_release_msg(gss_msg);
}

static void gss_pipe_dentry_destroy(struct dentry *dir,
                struct rpc_pipe_dir_object *pdo)
{
        struct gss_pipe *gss_pipe = pdo->pdo_data;
        struct rpc_pipe *pipe = gss_pipe->pipe;

        if (pipe->dentry != NULL) {
                rpc_unlink(pipe->dentry);
                pipe->dentry = NULL;
        }
}

static int gss_pipe_dentry_create(struct dentry *dir,
                struct rpc_pipe_dir_object *pdo)
{
        struct gss_pipe *p = pdo->pdo_data;
        struct dentry *dentry;

        dentry = rpc_mkpipe_dentry(dir, p->name, p->clnt, p->pipe);
        if (IS_ERR(dentry))
                return PTR_ERR(dentry);
        p->pipe->dentry = dentry;
        return 0;
}

static const struct rpc_pipe_dir_object_ops gss_pipe_dir_object_ops = {
        .create = gss_pipe_dentry_create,
        .destroy = gss_pipe_dentry_destroy,
};

static struct gss_pipe *gss_pipe_alloc(struct rpc_clnt *clnt,
                const char *name,
                const struct rpc_pipe_ops *upcall_ops)
{
        struct gss_pipe *p;
        int err = -ENOMEM;

        p = kmalloc(sizeof(*p), GFP_KERNEL);
        if (p == NULL)
                goto err;
        p->pipe = rpc_mkpipe_data(upcall_ops, RPC_PIPE_WAIT_FOR_OPEN);
        if (IS_ERR(p->pipe)) {
                err = PTR_ERR(p->pipe);
                goto err_free_gss_pipe;
        }
        p->name = name;
        p->clnt = clnt;
        kref_init(&p->kref);
        rpc_init_pipe_dir_object(&p->pdo,
                        &gss_pipe_dir_object_ops,
                        p);
        return p;
err_free_gss_pipe:
        kfree(p);
err:
        return ERR_PTR(err);
}

struct gss_alloc_pdo {
        struct rpc_clnt *clnt;
        const char *name;
        const struct rpc_pipe_ops *upcall_ops;
};

static int gss_pipe_match_pdo(struct rpc_pipe_dir_object *pdo, void *data)
{
        struct gss_pipe *gss_pipe;
        struct gss_alloc_pdo *args = data;

        if (pdo->pdo_ops != &gss_pipe_dir_object_ops)
                return 0;
        gss_pipe = container_of(pdo, struct gss_pipe, pdo);
        if (strcmp(gss_pipe->name, args->name) != 0)
                return 0;
        if (!kref_get_unless_zero(&gss_pipe->kref))
                return 0;
        return 1;
}

static struct rpc_pipe_dir_object *gss_pipe_alloc_pdo(void *data)
{
        struct gss_pipe *gss_pipe;
        struct gss_alloc_pdo *args = data;

        gss_pipe = gss_pipe_alloc(args->clnt, args->name, args->upcall_ops);
        if (!IS_ERR(gss_pipe))
                return &gss_pipe->pdo;
        return NULL;
}

static struct gss_pipe *gss_pipe_get(struct rpc_clnt *clnt,
                const char *name,
                const struct rpc_pipe_ops *upcall_ops)
{
        struct net *net = rpc_net_ns(clnt);
        struct rpc_pipe_dir_object *pdo;
        struct gss_alloc_pdo args = {
                .clnt = clnt,
                .name = name,
                .upcall_ops = upcall_ops,
        };

        pdo = rpc_find_or_alloc_pipe_dir_object(net,
                        &clnt->cl_pipedir_objects,
                        gss_pipe_match_pdo,
                        gss_pipe_alloc_pdo,
                        &args);
        if (pdo != NULL)
                return container_of(pdo, struct gss_pipe, pdo);
        return ERR_PTR(-ENOMEM);
}

static void __gss_pipe_free(struct gss_pipe *p)
{
        struct rpc_clnt *clnt = p->clnt;
        struct net *net = rpc_net_ns(clnt);

        rpc_remove_pipe_dir_object(net,
                        &clnt->cl_pipedir_objects,
                        &p->pdo);
        rpc_destroy_pipe_data(p->pipe);
        kfree(p);
}

static void __gss_pipe_release(struct kref *kref)
{
        struct gss_pipe *p = container_of(kref, struct gss_pipe, kref);

        __gss_pipe_free(p);
}

static void gss_pipe_free(struct gss_pipe *p)
{
        if (p != NULL)
                kref_put(&p->kref, __gss_pipe_release);
}

/*
 * NOTE: we have the opportunity to use different
 * parameters based on the input flavor (which must be a pseudoflavor)
 */
static struct gss_auth *
gss_create_new(const struct rpc_auth_create_args *args, struct rpc_clnt *clnt)
{
        rpc_authflavor_t flavor = args->pseudoflavor;
        struct gss_auth *gss_auth;
        struct gss_pipe *gss_pipe;
        struct rpc_auth * auth;
        int err = -ENOMEM; /* XXX? */

        if (!try_module_get(THIS_MODULE))
                return ERR_PTR(err);
        if (!(gss_auth = kmalloc(sizeof(*gss_auth), GFP_KERNEL)))
                goto out_dec;
        INIT_HLIST_NODE(&gss_auth->hash);
        gss_auth->target_name = NULL;
        if (args->target_name) {
                gss_auth->target_name = kstrdup(args->target_name, GFP_KERNEL);
                if (gss_auth->target_name == NULL)
                        goto err_free;
        }
        gss_auth->client = clnt;
        gss_auth->net = get_net_track(rpc_net_ns(clnt), &gss_auth->ns_tracker,
                                      GFP_KERNEL);
        err = -EINVAL;
        gss_auth->mech = gss_mech_get_by_pseudoflavor(flavor);
        if (!gss_auth->mech)
                goto err_put_net;
        gss_auth->service = gss_pseudoflavor_to_service(gss_auth->mech, flavor);
        if (gss_auth->service == 0)
                goto err_put_mech;
        if (!gssd_running(gss_auth->net))
                goto err_put_mech;
        auth = &gss_auth->rpc_auth;
        auth->au_cslack = GSS_CRED_SLACK >> 2;
        BUILD_BUG_ON(GSS_KRB5_MAX_SLACK_NEEDED > RPC_MAX_AUTH_SIZE);
        auth->au_rslack = GSS_KRB5_MAX_SLACK_NEEDED >> 2;
        auth->au_verfsize = GSS_VERF_SLACK >> 2;
        auth->au_ralign = GSS_VERF_SLACK >> 2;
        __set_bit(RPCAUTH_AUTH_UPDATE_SLACK, &auth->au_flags);
        auth->au_ops = &authgss_ops;
        auth->au_flavor = flavor;
        if (gss_pseudoflavor_to_datatouch(gss_auth->mech, flavor))
                __set_bit(RPCAUTH_AUTH_DATATOUCH, &auth->au_flags);
        refcount_set(&auth->au_count, 1);
        kref_init(&gss_auth->kref);

        err = rpcauth_init_credcache(auth);
        if (err)
                goto err_put_mech;
        /*
         * Note: if we created the old pipe first, then someone who
         * examined the directory at the right moment might conclude
         * that we supported only the old pipe.  So we instead create
         * the new pipe first.
         */
        gss_pipe = gss_pipe_get(clnt, "gssd", &gss_upcall_ops_v1);
        if (IS_ERR(gss_pipe)) {
                err = PTR_ERR(gss_pipe);
                goto err_destroy_credcache;
        }
        gss_auth->gss_pipe[1] = gss_pipe;

        gss_pipe = gss_pipe_get(clnt, gss_auth->mech->gm_name,
                        &gss_upcall_ops_v0);
        if (IS_ERR(gss_pipe)) {
                err = PTR_ERR(gss_pipe);
                goto err_destroy_pipe_1;
        }
        gss_auth->gss_pipe[0] = gss_pipe;

        return gss_auth;
err_destroy_pipe_1:
        gss_pipe_free(gss_auth->gss_pipe[1]);
err_destroy_credcache:
        rpcauth_destroy_credcache(auth);
err_put_mech:
        gss_mech_put(gss_auth->mech);
err_put_net:
        put_net_track(gss_auth->net, &gss_auth->ns_tracker);
err_free:
        kfree(gss_auth->target_name);
        kfree(gss_auth);
out_dec:
        module_put(THIS_MODULE);
        trace_rpcgss_createauth(flavor, err);
        return ERR_PTR(err);
}

static void
gss_free(struct gss_auth *gss_auth)
{
        gss_pipe_free(gss_auth->gss_pipe[0]);
        gss_pipe_free(gss_auth->gss_pipe[1]);
        gss_mech_put(gss_auth->mech);
        put_net_track(gss_auth->net, &gss_auth->ns_tracker);
        kfree(gss_auth->target_name);

        kfree(gss_auth);
        module_put(THIS_MODULE);
}

static void
gss_free_callback(struct kref *kref)
{
        struct gss_auth *gss_auth = container_of(kref, struct gss_auth, kref);

        gss_free(gss_auth);
}

static void
gss_put_auth(struct gss_auth *gss_auth)
{
        kref_put(&gss_auth->kref, gss_free_callback);
}

static void
gss_destroy(struct rpc_auth *auth)
{
        struct gss_auth *gss_auth = container_of(auth,
                        struct gss_auth, rpc_auth);

        if (hash_hashed(&gss_auth->hash)) {
                spin_lock(&gss_auth_hash_lock);
                hash_del(&gss_auth->hash);
                spin_unlock(&gss_auth_hash_lock);
        }

        gss_pipe_free(gss_auth->gss_pipe[0]);
        gss_auth->gss_pipe[0] = NULL;
        gss_pipe_free(gss_auth->gss_pipe[1]);
        gss_auth->gss_pipe[1] = NULL;
        rpcauth_destroy_credcache(auth);

        gss_put_auth(gss_auth);
}

/*
 * Auths may be shared between rpc clients that were cloned from a
 * common client with the same xprt, if they also share the flavor and
 * target_name.
 *
 * The auth is looked up from the oldest parent sharing the same
 * cl_xprt, and the auth itself references only that common parent
 * (which is guaranteed to last as long as any of its descendants).
 */
static struct gss_auth *
gss_auth_find_or_add_hashed(const struct rpc_auth_create_args *args,
                struct rpc_clnt *clnt,
                struct gss_auth *new)
{
        struct gss_auth *gss_auth;
        unsigned long hashval = (unsigned long)clnt;

        spin_lock(&gss_auth_hash_lock);
        hash_for_each_possible(gss_auth_hash_table,
                        gss_auth,
                        hash,
                        hashval) {
                if (gss_auth->client != clnt)
                        continue;
                if (gss_auth->rpc_auth.au_flavor != args->pseudoflavor)
                        continue;
                if (gss_auth->target_name != args->target_name) {
                        if (gss_auth->target_name == NULL)
                                continue;
                        if (args->target_name == NULL)
                                continue;
                        if (strcmp(gss_auth->target_name, args->target_name))
                                continue;
                }
                if (!refcount_inc_not_zero(&gss_auth->rpc_auth.au_count))
                        continue;
                goto out;
        }
        if (new)
                hash_add(gss_auth_hash_table, &new->hash, hashval);
        gss_auth = new;
out:
        spin_unlock(&gss_auth_hash_lock);
        return gss_auth;
}

static struct gss_auth *
gss_create_hashed(const struct rpc_auth_create_args *args,
                  struct rpc_clnt *clnt)
{
        struct gss_auth *gss_auth;
        struct gss_auth *new;

        gss_auth = gss_auth_find_or_add_hashed(args, clnt, NULL);
        if (gss_auth != NULL)
                goto out;
        new = gss_create_new(args, clnt);
        if (IS_ERR(new))
                return new;
        gss_auth = gss_auth_find_or_add_hashed(args, clnt, new);
        if (gss_auth != new)
                gss_destroy(&new->rpc_auth);
out:
        return gss_auth;
}

static struct rpc_auth *
gss_create(const struct rpc_auth_create_args *args, struct rpc_clnt *clnt)
{
        struct gss_auth *gss_auth;
        struct rpc_xprt_switch *xps = rcu_access_pointer(clnt->cl_xpi.xpi_xpswitch);

        while (clnt != clnt->cl_parent) {
                struct rpc_clnt *parent = clnt->cl_parent;
                /* Find the original parent for this transport */
                if (rcu_access_pointer(parent->cl_xpi.xpi_xpswitch) != xps)
                        break;
                clnt = parent;
        }

        gss_auth = gss_create_hashed(args, clnt);
        if (IS_ERR(gss_auth))
                return ERR_CAST(gss_auth);
        return &gss_auth->rpc_auth;
}

static struct gss_cred *
gss_dup_cred(struct gss_auth *gss_auth, struct gss_cred *gss_cred)
{
        struct gss_cred *new;

        /* Make a copy of the cred so that we can reference count it */
        new = kzalloc(sizeof(*gss_cred), GFP_KERNEL);
        if (new) {
                struct auth_cred acred = {
                        .cred = gss_cred->gc_base.cr_cred,
                };
                struct gss_cl_ctx *ctx =
                        rcu_dereference_protected(gss_cred->gc_ctx, 1);

                rpcauth_init_cred(&new->gc_base, &acred,
                                &gss_auth->rpc_auth,
                                &gss_nullops);
                new->gc_base.cr_flags = 1UL << RPCAUTH_CRED_UPTODATE;
                new->gc_service = gss_cred->gc_service;
                new->gc_principal = gss_cred->gc_principal;
                kref_get(&gss_auth->kref);
                rcu_assign_pointer(new->gc_ctx, ctx);
                gss_get_ctx(ctx);
        }
        return new;
}

/*
 * gss_send_destroy_context will cause the RPCSEC_GSS to send a NULL RPC call
 * to the server with the GSS control procedure field set to
 * RPC_GSS_PROC_DESTROY. This should normally cause the server to release
 * all RPCSEC_GSS state associated with that context.
 */
static void
gss_send_destroy_context(struct rpc_cred *cred)
{
        struct gss_cred *gss_cred = container_of(cred, struct gss_cred, gc_base);
        struct gss_auth *gss_auth = container_of(cred->cr_auth, struct gss_auth, rpc_auth);
        struct gss_cl_ctx *ctx = rcu_dereference_protected(gss_cred->gc_ctx, 1);
        struct gss_cred *new;
        struct rpc_task *task;

        new = gss_dup_cred(gss_auth, gss_cred);
        if (new) {
                ctx->gc_proc = RPC_GSS_PROC_DESTROY;

                trace_rpcgss_ctx_destroy(gss_cred);
                task = rpc_call_null(gss_auth->client, &new->gc_base,
                                     RPC_TASK_ASYNC);
                if (!IS_ERR(task))
                        rpc_put_task(task);

                put_rpccred(&new->gc_base);
        }
}

/* gss_destroy_cred (and gss_free_ctx) are used to clean up after failure
 * to create a new cred or context, so they check that things have been
 * allocated before freeing them. */
static void
gss_do_free_ctx(struct gss_cl_ctx *ctx)
{
        gss_delete_sec_context(&ctx->gc_gss_ctx);
        kfree(ctx->gc_wire_ctx.data);
        kfree(ctx->gc_acceptor.data);
        kfree(ctx);
}

static void
gss_free_ctx_callback(struct rcu_head *head)
{
        struct gss_cl_ctx *ctx = container_of(head, struct gss_cl_ctx, gc_rcu);
        gss_do_free_ctx(ctx);
}

static void
gss_free_ctx(struct gss_cl_ctx *ctx)
{
        call_rcu(&ctx->gc_rcu, gss_free_ctx_callback);
}

static void
gss_free_cred(struct gss_cred *gss_cred)
{
        kfree(gss_cred);
}

static void
gss_free_cred_callback(struct rcu_head *head)
{
        struct gss_cred *gss_cred = container_of(head, struct gss_cred, gc_base.cr_rcu);
        gss_free_cred(gss_cred);
}

static void
gss_destroy_nullcred(struct rpc_cred *cred)
{
        struct gss_cred *gss_cred = container_of(cred, struct gss_cred, gc_base);
        struct gss_auth *gss_auth = container_of(cred->cr_auth, struct gss_auth, rpc_auth);
        struct gss_cl_ctx *ctx = rcu_dereference_protected(gss_cred->gc_ctx, 1);

        RCU_INIT_POINTER(gss_cred->gc_ctx, NULL);
        put_cred(cred->cr_cred);
        call_rcu(&cred->cr_rcu, gss_free_cred_callback);
        if (ctx)
                gss_put_ctx(ctx);
        gss_put_auth(gss_auth);
}

static void
gss_destroy_cred(struct rpc_cred *cred)
{
        if (test_and_clear_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) != 0)
                gss_send_destroy_context(cred);
        gss_destroy_nullcred(cred);
}

static int
gss_hash_cred(struct auth_cred *acred, unsigned int hashbits)
{
        return hash_64(from_kuid(&init_user_ns, acred->cred->fsuid), hashbits);
}

/*
 * Lookup RPCSEC_GSS cred for the current process
 */
static struct rpc_cred *gss_lookup_cred(struct rpc_auth *auth,
                                        struct auth_cred *acred, int flags)
{
        return rpcauth_lookup_credcache(auth, acred, flags,
                                        rpc_task_gfp_mask());
}

static struct rpc_cred *
gss_create_cred(struct rpc_auth *auth, struct auth_cred *acred, int flags, gfp_t gfp)
{
        struct gss_auth *gss_auth = container_of(auth, struct gss_auth, rpc_auth);
        struct gss_cred        *cred = NULL;
        int err = -ENOMEM;

        if (!(cred = kzalloc(sizeof(*cred), gfp)))
                goto out_err;

        rpcauth_init_cred(&cred->gc_base, acred, auth, &gss_credops);
        /*
         * Note: in order to force a call to call_refresh(), we deliberately
         * fail to flag the credential as RPCAUTH_CRED_UPTODATE.
         */
        cred->gc_base.cr_flags = 1UL << RPCAUTH_CRED_NEW;
        cred->gc_service = gss_auth->service;
        cred->gc_principal = acred->principal;
        kref_get(&gss_auth->kref);
        return &cred->gc_base;

out_err:
        return ERR_PTR(err);
}

static int
gss_cred_init(struct rpc_auth *auth, struct rpc_cred *cred)
{
        struct gss_auth *gss_auth = container_of(auth, struct gss_auth, rpc_auth);
        struct gss_cred *gss_cred = container_of(cred,struct gss_cred, gc_base);
        int err;

        do {
                err = gss_create_upcall(gss_auth, gss_cred);
        } while (err == -EAGAIN);
        return err;
}

static char *
gss_stringify_acceptor(struct rpc_cred *cred)
{
        char *string = NULL;
        struct gss_cred *gss_cred = container_of(cred, struct gss_cred, gc_base);
        struct gss_cl_ctx *ctx;
        unsigned int len;
        struct xdr_netobj *acceptor;

        rcu_read_lock();
        ctx = rcu_dereference(gss_cred->gc_ctx);
        if (!ctx)
                goto out;

        len = ctx->gc_acceptor.len;
        rcu_read_unlock();

        /* no point if there's no string */
        if (!len)
                return NULL;
realloc:
        string = kmalloc(len + 1, GFP_KERNEL);
        if (!string)
                return NULL;

        rcu_read_lock();
        ctx = rcu_dereference(gss_cred->gc_ctx);

        /* did the ctx disappear or was it replaced by one with no acceptor? */
        if (!ctx || !ctx->gc_acceptor.len) {
                kfree(string);
                string = NULL;
                goto out;
        }

        acceptor = &ctx->gc_acceptor;

        /*
         * Did we find a new acceptor that's longer than the original? Allocate
         * a longer buffer and try again.
         */
        if (len < acceptor->len) {
                len = acceptor->len;
                rcu_read_unlock();
                kfree(string);
                goto realloc;
        }

        memcpy(string, acceptor->data, acceptor->len);
        string[acceptor->len] = '\0';
out:
        rcu_read_unlock();
        return string;
}

/*
 * Returns -EACCES if GSS context is NULL or will expire within the
 * timeout (miliseconds)
 */
static int
gss_key_timeout(struct rpc_cred *rc)
{
        struct gss_cred *gss_cred = container_of(rc, struct gss_cred, gc_base);
        struct gss_cl_ctx *ctx;
        unsigned long timeout = jiffies + (gss_key_expire_timeo * HZ);
        int ret = 0;

        rcu_read_lock();
        ctx = rcu_dereference(gss_cred->gc_ctx);
        if (!ctx || time_after(timeout, ctx->gc_expiry))
                ret = -EACCES;
        rcu_read_unlock();

        return ret;
}

static int
gss_match(struct auth_cred *acred, struct rpc_cred *rc, int flags)
{
        struct gss_cred *gss_cred = container_of(rc, struct gss_cred, gc_base);
        struct gss_cl_ctx *ctx;
        int ret;

        if (test_bit(RPCAUTH_CRED_NEW, &rc->cr_flags))
                goto out;
        /* Don't match with creds that have expired. */
        rcu_read_lock();
        ctx = rcu_dereference(gss_cred->gc_ctx);
        if (!ctx || time_after(jiffies, ctx->gc_expiry)) {
                rcu_read_unlock();
                return 0;
        }
        rcu_read_unlock();
        if (!test_bit(RPCAUTH_CRED_UPTODATE, &rc->cr_flags))
                return 0;
out:
        if (acred->principal != NULL) {
                if (gss_cred->gc_principal == NULL)
                        return 0;
                ret = strcmp(acred->principal, gss_cred->gc_principal) == 0;
        } else {
                if (gss_cred->gc_principal != NULL)
                        return 0;
                ret = uid_eq(rc->cr_cred->fsuid, acred->cred->fsuid);
        }
        return ret;
}

/*
 * Marshal credentials.
 *
 * The expensive part is computing the verifier. We can't cache a
 * pre-computed version of the verifier because the seqno, which
 * is different every time, is included in the MIC.
 */
static int gss_marshal(struct rpc_task *task, struct xdr_stream *xdr)
{
        struct rpc_rqst *req = task->tk_rqstp;
        struct rpc_cred *cred = req->rq_cred;
        struct gss_cred        *gss_cred = container_of(cred, struct gss_cred,
                                                 gc_base);
        struct gss_cl_ctx        *ctx = gss_cred_get_ctx(cred);
        __be32                *p, *cred_len;
        u32             maj_stat = 0;
        struct xdr_netobj mic;
        struct kvec        iov;
        struct xdr_buf        verf_buf;
        int status;

        /* Credential */

        p = xdr_reserve_space(xdr, 7 * sizeof(*p) +
                              ctx->gc_wire_ctx.len);
        if (!p)
                goto marshal_failed;
        *p++ = rpc_auth_gss;
        cred_len = p++;

        spin_lock(&ctx->gc_seq_lock);
        req->rq_seqno = (ctx->gc_seq < MAXSEQ) ? ctx->gc_seq++ : MAXSEQ;
        spin_unlock(&ctx->gc_seq_lock);
        if (req->rq_seqno == MAXSEQ)
                goto expired;
        trace_rpcgss_seqno(task);

        *p++ = cpu_to_be32(RPC_GSS_VERSION);
        *p++ = cpu_to_be32(ctx->gc_proc);
        *p++ = cpu_to_be32(req->rq_seqno);
        *p++ = cpu_to_be32(gss_cred->gc_service);
        p = xdr_encode_netobj(p, &ctx->gc_wire_ctx);
        *cred_len = cpu_to_be32((p - (cred_len + 1)) << 2);

        /* Verifier */

        /* We compute the checksum for the verifier over the xdr-encoded bytes
         * starting with the xid and ending at the end of the credential: */
        iov.iov_base = req->rq_snd_buf.head[0].iov_base;
        iov.iov_len = (u8 *)p - (u8 *)iov.iov_base;
        xdr_buf_from_iov(&iov, &verf_buf);

        p = xdr_reserve_space(xdr, sizeof(*p));
        if (!p)
                goto marshal_failed;
        *p++ = rpc_auth_gss;
        mic.data = (u8 *)(p + 1);
        maj_stat = gss_get_mic(ctx->gc_gss_ctx, &verf_buf, &mic);
        if (maj_stat == GSS_S_CONTEXT_EXPIRED)
                goto expired;
        else if (maj_stat != 0)
                goto bad_mic;
        if (xdr_stream_encode_opaque_inline(xdr, (void **)&p, mic.len) < 0)
                goto marshal_failed;
        status = 0;
out:
        gss_put_ctx(ctx);
        return status;
expired:
        clear_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags);
        status = -EKEYEXPIRED;
        goto out;
marshal_failed:
        status = -EMSGSIZE;
        goto out;
bad_mic:
        trace_rpcgss_get_mic(task, maj_stat);
        status = -EIO;
        goto out;
}

static int gss_renew_cred(struct rpc_task *task)
{
        struct rpc_cred *oldcred = task->tk_rqstp->rq_cred;
        struct gss_cred *gss_cred = container_of(oldcred,
                                                 struct gss_cred,
                                                 gc_base);
        struct rpc_auth *auth = oldcred->cr_auth;
        struct auth_cred acred = {
                .cred = oldcred->cr_cred,
                .principal = gss_cred->gc_principal,
        };
        struct rpc_cred *new;

        new = gss_lookup_cred(auth, &acred, RPCAUTH_LOOKUP_NEW);
        if (IS_ERR(new))
                return PTR_ERR(new);

        task->tk_rqstp->rq_cred = new;
        put_rpccred(oldcred);
        return 0;
}

static int gss_cred_is_negative_entry(struct rpc_cred *cred)
{
        if (test_bit(RPCAUTH_CRED_NEGATIVE, &cred->cr_flags)) {
                unsigned long now = jiffies;
                unsigned long begin, expire;
                struct gss_cred *gss_cred;

                gss_cred = container_of(cred, struct gss_cred, gc_base);
                begin = gss_cred->gc_upcall_timestamp;
                expire = begin + gss_expired_cred_retry_delay * HZ;

                if (time_in_range_open(now, begin, expire))
                        return 1;
        }
        return 0;
}

/*
* Refresh credentials. XXX - finish
*/
static int
gss_refresh(struct rpc_task *task)
{
        struct rpc_cred *cred = task->tk_rqstp->rq_cred;
        int ret = 0;

        if (gss_cred_is_negative_entry(cred))
                return -EKEYEXPIRED;

        if (!test_bit(RPCAUTH_CRED_NEW, &cred->cr_flags) &&
                        !test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags)) {
                ret = gss_renew_cred(task);
                if (ret < 0)
                        goto out;
                cred = task->tk_rqstp->rq_cred;
        }

        if (test_bit(RPCAUTH_CRED_NEW, &cred->cr_flags))
                ret = gss_refresh_upcall(task);
out:
        return ret;
}

/* Dummy refresh routine: used only when destroying the context */
static int
gss_refresh_null(struct rpc_task *task)
{
        return 0;
}

static int
gss_validate(struct rpc_task *task, struct xdr_stream *xdr)
{
        struct rpc_cred *cred = task->tk_rqstp->rq_cred;
        struct gss_cl_ctx *ctx = gss_cred_get_ctx(cred);
        __be32                *p, *seq = NULL;
        struct kvec        iov;
        struct xdr_buf        verf_buf;
        struct xdr_netobj mic;
        u32                len, maj_stat;
        int                status;

        p = xdr_inline_decode(xdr, 2 * sizeof(*p));
        if (!p)
                goto validate_failed;
        if (*p++ != rpc_auth_gss)
                goto validate_failed;
        len = be32_to_cpup(p);
        if (len > RPC_MAX_AUTH_SIZE)
                goto validate_failed;
        p = xdr_inline_decode(xdr, len);
        if (!p)
                goto validate_failed;

        seq = kmalloc(4, GFP_KERNEL);
        if (!seq)
                goto validate_failed;
        *seq = cpu_to_be32(task->tk_rqstp->rq_seqno);
        iov.iov_base = seq;
        iov.iov_len = 4;
        xdr_buf_from_iov(&iov, &verf_buf);
        mic.data = (u8 *)p;
        mic.len = len;
        maj_stat = gss_verify_mic(ctx->gc_gss_ctx, &verf_buf, &mic);
        if (maj_stat == GSS_S_CONTEXT_EXPIRED)
                clear_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags);
        if (maj_stat)
                goto bad_mic;

        /* We leave it to unwrap to calculate au_rslack. For now we just
         * calculate the length of the verifier: */
        if (test_bit(RPCAUTH_AUTH_UPDATE_SLACK, &cred->cr_auth->au_flags))
                cred->cr_auth->au_verfsize = XDR_QUADLEN(len) + 2;
        status = 0;
out:
        gss_put_ctx(ctx);
        kfree(seq);
        return status;

validate_failed:
        status = -EIO;
        goto out;
bad_mic:
        trace_rpcgss_verify_mic(task, maj_stat);
        status = -EACCES;
        goto out;
}

static noinline_for_stack int
gss_wrap_req_integ(struct rpc_cred *cred, struct gss_cl_ctx *ctx,
                   struct rpc_task *task, struct xdr_stream *xdr)
{
        struct rpc_rqst *rqstp = task->tk_rqstp;
        struct xdr_buf integ_buf, *snd_buf = &rqstp->rq_snd_buf;
        struct xdr_netobj mic;
        __be32 *p, *integ_len;
        u32 offset, maj_stat;

        p = xdr_reserve_space(xdr, 2 * sizeof(*p));
        if (!p)
                goto wrap_failed;
        integ_len = p++;
        *p = cpu_to_be32(rqstp->rq_seqno);

        if (rpcauth_wrap_req_encode(task, xdr))
                goto wrap_failed;

        offset = (u8 *)p - (u8 *)snd_buf->head[0].iov_base;
        if (xdr_buf_subsegment(snd_buf, &integ_buf,
                                offset, snd_buf->len - offset))
                goto wrap_failed;
        *integ_len = cpu_to_be32(integ_buf.len);

        p = xdr_reserve_space(xdr, 0);
        if (!p)
                goto wrap_failed;
        mic.data = (u8 *)(p + 1);
        maj_stat = gss_get_mic(ctx->gc_gss_ctx, &integ_buf, &mic);
        if (maj_stat == GSS_S_CONTEXT_EXPIRED)
                clear_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags);
        else if (maj_stat)
                goto bad_mic;
        /* Check that the trailing MIC fit in the buffer, after the fact */
        if (xdr_stream_encode_opaque_inline(xdr, (void **)&p, mic.len) < 0)
                goto wrap_failed;
        return 0;
wrap_failed:
        return -EMSGSIZE;
bad_mic:
        trace_rpcgss_get_mic(task, maj_stat);
        return -EIO;
}

static void
priv_release_snd_buf(struct rpc_rqst *rqstp)
{
        int i;

        for (i=0; i < rqstp->rq_enc_pages_num; i++)
                __free_page(rqstp->rq_enc_pages[i]);
        kfree(rqstp->rq_enc_pages);
        rqstp->rq_release_snd_buf = NULL;
}

static int
alloc_enc_pages(struct rpc_rqst *rqstp)
{
        struct xdr_buf *snd_buf = &rqstp->rq_snd_buf;
        int first, last, i;

        if (rqstp->rq_release_snd_buf)
                rqstp->rq_release_snd_buf(rqstp);

        if (snd_buf->page_len == 0) {
                rqstp->rq_enc_pages_num = 0;
                return 0;
        }

        first = snd_buf->page_base >> PAGE_SHIFT;
        last = (snd_buf->page_base + snd_buf->page_len - 1) >> PAGE_SHIFT;
        rqstp->rq_enc_pages_num = last - first + 1 + 1;
        rqstp->rq_enc_pages
                = kmalloc_array(rqstp->rq_enc_pages_num,
                                sizeof(struct page *),
                                GFP_KERNEL);
        if (!rqstp->rq_enc_pages)
                goto out;
        for (i=0; i < rqstp->rq_enc_pages_num; i++) {
                rqstp->rq_enc_pages[i] = alloc_page(GFP_KERNEL);
                if (rqstp->rq_enc_pages[i] == NULL)
                        goto out_free;
        }
        rqstp->rq_release_snd_buf = priv_release_snd_buf;
        return 0;
out_free:
        rqstp->rq_enc_pages_num = i;
        priv_release_snd_buf(rqstp);
out:
        return -EAGAIN;
}

static noinline_for_stack int
gss_wrap_req_priv(struct rpc_cred *cred, struct gss_cl_ctx *ctx,
                  struct rpc_task *task, struct xdr_stream *xdr)
{
        struct rpc_rqst *rqstp = task->tk_rqstp;
        struct xdr_buf        *snd_buf = &rqstp->rq_snd_buf;
        u32                pad, offset, maj_stat;
        int                status;
        __be32                *p, *opaque_len;
        struct page        **inpages;
        int                first;
        struct kvec        *iov;

        status = -EIO;
        p = xdr_reserve_space(xdr, 2 * sizeof(*p));
        if (!p)
                goto wrap_failed;
        opaque_len = p++;
        *p = cpu_to_be32(rqstp->rq_seqno);

        if (rpcauth_wrap_req_encode(task, xdr))
                goto wrap_failed;

        status = alloc_enc_pages(rqstp);
        if (unlikely(status))
                goto wrap_failed;
        first = snd_buf->page_base >> PAGE_SHIFT;
        inpages = snd_buf->pages + first;
        snd_buf->pages = rqstp->rq_enc_pages;
        snd_buf->page_base -= first << PAGE_SHIFT;
        /*
         * Move the tail into its own page, in case gss_wrap needs
         * more space in the head when wrapping.
         *
         * Still... Why can't gss_wrap just slide the tail down?
         */
        if (snd_buf->page_len || snd_buf->tail[0].iov_len) {
                char *tmp;

                tmp = page_address(rqstp->rq_enc_pages[rqstp->rq_enc_pages_num - 1]);
                memcpy(tmp, snd_buf->tail[0].iov_base, snd_buf->tail[0].iov_len);
                snd_buf->tail[0].iov_base = tmp;
        }
        offset = (u8 *)p - (u8 *)snd_buf->head[0].iov_base;
        maj_stat = gss_wrap(ctx->gc_gss_ctx, offset, snd_buf, inpages);
        /* slack space should prevent this ever happening: */
        if (unlikely(snd_buf->len > snd_buf->buflen)) {
                status = -EIO;
                goto wrap_failed;
        }
        /* We're assuming that when GSS_S_CONTEXT_EXPIRED, the encryption was
         * done anyway, so it's safe to put the request on the wire: */
        if (maj_stat == GSS_S_CONTEXT_EXPIRED)
                clear_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags);
        else if (maj_stat)
                goto bad_wrap;

        *opaque_len = cpu_to_be32(snd_buf->len - offset);
        /* guess whether the pad goes into the head or the tail: */
        if (snd_buf->page_len || snd_buf->tail[0].iov_len)
                iov = snd_buf->tail;
        else
                iov = snd_buf->head;
        p = iov->iov_base + iov->iov_len;
        pad = xdr_pad_size(snd_buf->len - offset);
        memset(p, 0, pad);
        iov->iov_len += pad;
        snd_buf->len += pad;

        return 0;
wrap_failed:
        return status;
bad_wrap:
        trace_rpcgss_wrap(task, maj_stat);
        return -EIO;
}

static int gss_wrap_req(struct rpc_task *task, struct xdr_stream *xdr)
{
        struct rpc_cred *cred = task->tk_rqstp->rq_cred;
        struct gss_cred        *gss_cred = container_of(cred, struct gss_cred,
                        gc_base);
        struct gss_cl_ctx *ctx = gss_cred_get_ctx(cred);
        int status;

        status = -EIO;
        if (ctx->gc_proc != RPC_GSS_PROC_DATA) {
                /* The spec seems a little ambiguous here, but I think that not
                 * wrapping context destruction requests makes the most sense.
                 */
                status = rpcauth_wrap_req_encode(task, xdr);
                goto out;
        }
        switch (gss_cred->gc_service) {
        case RPC_GSS_SVC_NONE:
                status = rpcauth_wrap_req_encode(task, xdr);
                break;
        case RPC_GSS_SVC_INTEGRITY:
                status = gss_wrap_req_integ(cred, ctx, task, xdr);
                break;
        case RPC_GSS_SVC_PRIVACY:
                status = gss_wrap_req_priv(cred, ctx, task, xdr);
                break;
        default:
                status = -EIO;
        }
out:
        gss_put_ctx(ctx);
        return status;
}

/**
 * gss_update_rslack - Possibly update RPC receive buffer size estimates
 * @task: rpc_task for incoming RPC Reply being unwrapped
 * @cred: controlling rpc_cred for @task
 * @before: XDR words needed before each RPC Reply message
 * @after: XDR words needed following each RPC Reply message
 *
 */
static void gss_update_rslack(struct rpc_task *task, struct rpc_cred *cred,
                              unsigned int before, unsigned int after)
{
        struct rpc_auth *auth = cred->cr_auth;

        if (test_and_clear_bit(RPCAUTH_AUTH_UPDATE_SLACK, &auth->au_flags)) {
                auth->au_ralign = auth->au_verfsize + before;
                auth->au_rslack = auth->au_verfsize + after;
                trace_rpcgss_update_slack(task, auth);
        }
}

static int
gss_unwrap_resp_auth(struct rpc_task *task, struct rpc_cred *cred)
{
        gss_update_rslack(task, cred, 0, 0);
        return 0;
}

/*
 * RFC 2203, Section 5.3.2.2
 *
 *        struct rpc_gss_integ_data {
 *                opaque databody_integ<>;
 *                opaque checksum<>;
 *        };
 *
 *        struct rpc_gss_data_t {
 *                unsigned int seq_num;
 *                proc_req_arg_t arg;
 *        };
 */
static noinline_for_stack int
gss_unwrap_resp_integ(struct rpc_task *task, struct rpc_cred *cred,
                      struct gss_cl_ctx *ctx, struct rpc_rqst *rqstp,
                      struct xdr_stream *xdr)
{
        struct xdr_buf gss_data, *rcv_buf = &rqstp->rq_rcv_buf;
        u32 len, offset, seqno, maj_stat;
        struct xdr_netobj mic;
        int ret;

        ret = -EIO;
        mic.data = NULL;

        /* opaque databody_integ<>; */
        if (xdr_stream_decode_u32(xdr, &len))
                goto unwrap_failed;
        if (len & 3)
                goto unwrap_failed;
        offset = rcv_buf->len - xdr_stream_remaining(xdr);
        if (xdr_stream_decode_u32(xdr, &seqno))
                goto unwrap_failed;
        if (seqno != rqstp->rq_seqno)
                goto bad_seqno;
        if (xdr_buf_subsegment(rcv_buf, &gss_data, offset, len))
                goto unwrap_failed;

        /*
         * The xdr_stream now points to the beginning of the
         * upper layer payload, to be passed below to
         * rpcauth_unwrap_resp_decode(). The checksum, which
         * follows the upper layer payload in @rcv_buf, is
         * located and parsed without updating the xdr_stream.
         */

        /* opaque checksum<>; */
        offset += len;
        if (xdr_decode_word(rcv_buf, offset, &len))
                goto unwrap_failed;
        offset += sizeof(__be32);
        if (offset + len > rcv_buf->len)
                goto unwrap_failed;
        mic.len = len;
        mic.data = kmalloc(len, GFP_KERNEL);
        if (ZERO_OR_NULL_PTR(mic.data))
                goto unwrap_failed;
        if (read_bytes_from_xdr_buf(rcv_buf, offset, mic.data, mic.len))
                goto unwrap_failed;

        maj_stat = gss_verify_mic(ctx->gc_gss_ctx, &gss_data, &mic);
        if (maj_stat == GSS_S_CONTEXT_EXPIRED)
                clear_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags);
        if (maj_stat != GSS_S_COMPLETE)
                goto bad_mic;

        gss_update_rslack(task, cred, 2, 2 + 1 + XDR_QUADLEN(mic.len));
        ret = 0;

out:
        kfree(mic.data);
        return ret;

unwrap_failed:
        trace_rpcgss_unwrap_failed(task);
        goto out;
bad_seqno:
        trace_rpcgss_bad_seqno(task, rqstp->rq_seqno, seqno);
        goto out;
bad_mic:
        trace_rpcgss_verify_mic(task, maj_stat);
        goto out;
}

static noinline_for_stack int
gss_unwrap_resp_priv(struct rpc_task *task, struct rpc_cred *cred,
                     struct gss_cl_ctx *ctx, struct rpc_rqst *rqstp,
                     struct xdr_stream *xdr)
{
        struct xdr_buf *rcv_buf = &rqstp->rq_rcv_buf;
        struct kvec *head = rqstp->rq_rcv_buf.head;
        u32 offset, opaque_len, maj_stat;
        __be32 *p;

        p = xdr_inline_decode(xdr, 2 * sizeof(*p));
        if (unlikely(!p))
                goto unwrap_failed;
        opaque_len = be32_to_cpup(p++);
        offset = (u8 *)(p) - (u8 *)head->iov_base;
        if (offset + opaque_len > rcv_buf->len)
                goto unwrap_failed;

        maj_stat = gss_unwrap(ctx->gc_gss_ctx, offset,
                              offset + opaque_len, rcv_buf);
        if (maj_stat == GSS_S_CONTEXT_EXPIRED)
                clear_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags);
        if (maj_stat != GSS_S_COMPLETE)
                goto bad_unwrap;
        /* gss_unwrap decrypted the sequence number */
        if (be32_to_cpup(p++) != rqstp->rq_seqno)
                goto bad_seqno;

        /* gss_unwrap redacts the opaque blob from the head iovec.
         * rcv_buf has changed, thus the stream needs to be reset.
         */
        xdr_init_decode(xdr, rcv_buf, p, rqstp);

        gss_update_rslack(task, cred, 2 + ctx->gc_gss_ctx->align,
                          2 + ctx->gc_gss_ctx->slack);

        return 0;
unwrap_failed:
        trace_rpcgss_unwrap_failed(task);
        return -EIO;
bad_seqno:
        trace_rpcgss_bad_seqno(task, rqstp->rq_seqno, be32_to_cpup(--p));
        return -EIO;
bad_unwrap:
        trace_rpcgss_unwrap(task, maj_stat);
        return -EIO;
}

static bool
gss_seq_is_newer(u32 new, u32 old)
{
        return (s32)(new - old) > 0;
}

static bool
gss_xmit_need_reencode(struct rpc_task *task)
{
        struct rpc_rqst *req = task->tk_rqstp;
        struct rpc_cred *cred = req->rq_cred;
        struct gss_cl_ctx *ctx = gss_cred_get_ctx(cred);
        u32 win, seq_xmit = 0;
        bool ret = true;

        if (!ctx)
                goto out;

        if (gss_seq_is_newer(req->rq_seqno, READ_ONCE(ctx->gc_seq)))
                goto out_ctx;

        seq_xmit = READ_ONCE(ctx->gc_seq_xmit);
        while (gss_seq_is_newer(req->rq_seqno, seq_xmit)) {
                u32 tmp = seq_xmit;

                seq_xmit = cmpxchg(&ctx->gc_seq_xmit, tmp, req->rq_seqno);
                if (seq_xmit == tmp) {
                        ret = false;
                        goto out_ctx;
                }
        }

        win = ctx->gc_win;
        if (win > 0)
                ret = !gss_seq_is_newer(req->rq_seqno, seq_xmit - win);

out_ctx:
        gss_put_ctx(ctx);
out:
        trace_rpcgss_need_reencode(task, seq_xmit, ret);
        return ret;
}

static int
gss_unwrap_resp(struct rpc_task *task, struct xdr_stream *xdr)
{
        struct rpc_rqst *rqstp = task->tk_rqstp;
        struct rpc_cred *cred = rqstp->rq_cred;
        struct gss_cred *gss_cred = container_of(cred, struct gss_cred,
                        gc_base);
        struct gss_cl_ctx *ctx = gss_cred_get_ctx(cred);
        int status = -EIO;

        if (ctx->gc_proc != RPC_GSS_PROC_DATA)
                goto out_decode;
        switch (gss_cred->gc_service) {
        case RPC_GSS_SVC_NONE:
                status = gss_unwrap_resp_auth(task, cred);
                break;
        case RPC_GSS_SVC_INTEGRITY:
                status = gss_unwrap_resp_integ(task, cred, ctx, rqstp, xdr);
                break;
        case RPC_GSS_SVC_PRIVACY:
                status = gss_unwrap_resp_priv(task, cred, ctx, rqstp, xdr);
                break;
        }
        if (status)
                goto out;

out_decode:
        status = rpcauth_unwrap_resp_decode(task, xdr);
out:
        gss_put_ctx(ctx);
        return status;
}

static const struct rpc_authops authgss_ops = {
        .owner                = THIS_MODULE,
        .au_flavor        = RPC_AUTH_GSS,
        .au_name        = "RPCSEC_GSS",
        .create                = gss_create,
        .destroy        = gss_destroy,
        .hash_cred        = gss_hash_cred,
        .lookup_cred        = gss_lookup_cred,
        .crcreate        = gss_create_cred,
        .info2flavor        = gss_mech_info2flavor,
        .flavor2info        = gss_mech_flavor2info,
};

static const struct rpc_credops gss_credops = {
        .cr_name                = "AUTH_GSS",
        .crdestroy                = gss_destroy_cred,
        .cr_init                = gss_cred_init,
        .crmatch                = gss_match,
        .crmarshal                = gss_marshal,
        .crrefresh                = gss_refresh,
        .crvalidate                = gss_validate,
        .crwrap_req                = gss_wrap_req,
        .crunwrap_resp                = gss_unwrap_resp,
        .crkey_timeout                = gss_key_timeout,
        .crstringify_acceptor        = gss_stringify_acceptor,
        .crneed_reencode        = gss_xmit_need_reencode,
};

static const struct rpc_credops gss_nullops = {
        .cr_name                = "AUTH_GSS",
        .crdestroy                = gss_destroy_nullcred,
        .crmatch                = gss_match,
        .crmarshal                = gss_marshal,
        .crrefresh                = gss_refresh_null,
        .crvalidate                = gss_validate,
        .crwrap_req                = gss_wrap_req,
        .crunwrap_resp                = gss_unwrap_resp,
        .crstringify_acceptor        = gss_stringify_acceptor,
};

static const struct rpc_pipe_ops gss_upcall_ops_v0 = {
        .upcall                = gss_v0_upcall,
        .downcall        = gss_pipe_downcall,
        .destroy_msg        = gss_pipe_destroy_msg,
        .open_pipe        = gss_pipe_open_v0,
        .release_pipe        = gss_pipe_release,
};

static const struct rpc_pipe_ops gss_upcall_ops_v1 = {
        .upcall                = gss_v1_upcall,
        .downcall        = gss_pipe_downcall,
        .destroy_msg        = gss_pipe_destroy_msg,
        .open_pipe        = gss_pipe_open_v1,
        .release_pipe        = gss_pipe_release,
};

static __net_init int rpcsec_gss_init_net(struct net *net)
{
        return gss_svc_init_net(net);
}

static __net_exit void rpcsec_gss_exit_net(struct net *net)
{
        gss_svc_shutdown_net(net);
}

static struct pernet_operations rpcsec_gss_net_ops = {
        .init = rpcsec_gss_init_net,
        .exit = rpcsec_gss_exit_net,
};

/*
 * Initialize RPCSEC_GSS module
 */
static int __init init_rpcsec_gss(void)
{
        int err = 0;

        err = rpcauth_register(&authgss_ops);
        if (err)
                goto out;
        err = gss_svc_init();
        if (err)
                goto out_unregister;
        err = register_pernet_subsys(&rpcsec_gss_net_ops);
        if (err)
                goto out_svc_exit;
        rpc_init_wait_queue(&pipe_version_rpc_waitqueue, "gss pipe version");
        return 0;
out_svc_exit:
        gss_svc_shutdown();
out_unregister:
        rpcauth_unregister(&authgss_ops);
out:
        return err;
}

static void __exit exit_rpcsec_gss(void)
{
        unregister_pernet_subsys(&rpcsec_gss_net_ops);
        gss_svc_shutdown();
        rpcauth_unregister(&authgss_ops);
        rcu_barrier(); /* Wait for completion of call_rcu()'s */
}

MODULE_ALIAS("rpc-auth-6");
MODULE_DESCRIPTION("Sun RPC Kerberos RPCSEC_GSS client authentication");
MODULE_LICENSE("GPL");
module_param_named(expired_cred_retry_delay,
                   gss_expired_cred_retry_delay,
                   uint, 0644);
MODULE_PARM_DESC(expired_cred_retry_delay, "Timeout (in seconds) until "
                "the RPC engine retries an expired credential");

module_param_named(key_expire_timeo,
                   gss_key_expire_timeo,
                   uint, 0644);
MODULE_PARM_DESC(key_expire_timeo, "Time (in seconds) at the end of a "
                "credential keys lifetime where the NFS layer cleans up "
                "prior to key expiration");

module_init(init_rpcsec_gss)
module_exit(exit_rpcsec_gss)







































































































































































































   10 





















  859 
  859 







































































  860 












  860 
  860 

















  311 






















































































































































































































































































































































   11 










































































































































































































































































































































































































































  859 



























































































































































































































































































































































    3 


























  169 













































































































































































































































































































































  859 










  859 



  859 


  311 





   11 
   11 
























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































  101 























































































































  166 


























































































  166 















  166 



























  142 






























   10 










   60 























  169 


































   26 















  166 



























  169 



  169 


  169 


















  102 





  101 










































  102 






























  102 

























  860 





  841 
  102 







  859 


























































































































  860 












  169 






  859 






  859 













  860 
















  860 











  860 

    4 


















  859 

  860 





  860 





  859 





  109 





  833 


























































































  860 












  859 









  860 




























































  858 







  860 

  860 

  858 





























































































































































  860 









  169 
  859 









  183 





  858 



  858 






  859 








  858 












  858 












  859 











  183 

    3 



  835 
  101 
















  169 












    1 


  860 

















    3 





    2 
    3 

























  859 

  182 




  860 







  859 
  183 
















































  860 





















  169 





  167 
    2 








  859 


















  859 












  858 










  860 


  860 





























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
6195
6196
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
6284
6285
6286
6287
6288
6289
6290
6291
6292
6293
6294
6295
6296
6297
6298
6299
6300
6301
6302
6303
6304
6305
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
6380
6381
6382
6383
6384
6385
6386
6387
6388
6389
6390
6391
6392
6393
6394
6395
6396
6397
6398
6399
6400
6401
6402
6403
6404
6405
6406
6407
6408
6409
6410
6411
6412
6413
6414
6415
6416
6417
6418
6419
6420
6421
6422
6423
6424
6425
6426
6427
6428
6429
6430
6431
6432
6433
6434
6435
6436
6437
6438
6439
6440
6441
6442
6443
6444
6445
6446
6447
6448
6449
6450
6451
6452
6453
6454
6455
6456
6457
6458
6459
6460
6461
6462
6463
6464
6465
6466
6467
6468
6469
6470
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480
6481
6482
6483
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516
6517
6518
6519
6520
6521
6522
6523
6524
6525
6526
6527
6528
6529
6530
6531
6532
6533
6534
6535
6536
6537
6538
6539
6540
6541
6542
6543
6544
6545
6546
6547
6548
6549
6550
6551
6552
6553
6554
6555
6556
6557
6558
6559
6560
6561
6562
6563
6564
6565
6566
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576
6577
6578
6579
6580
6581
6582
6583
6584
6585
6586
6587
6588
6589
6590
6591
6592
6593
6594
6595
6596
6597
6598
6599
6600
6601
6602
6603
6604
6605
6606
6607
6608
6609
6610
6611
6612
6613
6614
6615
6616
6617
6618
6619
6620
6621
6622
6623
6624
6625
6626
6627
6628
6629
6630
6631
6632
6633
6634
6635
6636
6637
6638
6639
6640
6641
6642
6643
6644
6645
6646
6647
6648
6649
6650
6651
6652
6653
6654
6655
6656
6657
6658
6659
6660
6661
6662
6663
6664
6665
6666
6667
6668
6669
6670
6671
6672
6673
6674
6675
6676
6677
6678
6679
6680
6681
6682
6683
6684
6685
6686
6687
6688
6689
6690
6691
6692
6693
6694
6695
6696
6697
6698
6699
6700
6701
6702
6703
6704
6705
6706
6707
6708
6709
6710
6711
6712
6713
6714
6715
6716
6717
6718
6719
6720
6721
6722
6723
6724
6725
6726
6727
6728
6729
6730
6731
6732
6733
6734
6735
6736
6737
6738
6739
6740
6741
6742
6743
6744
6745
6746
6747
6748
6749
6750
6751
6752
6753
6754
6755
6756
6757
6758
6759
6760
6761
6762
6763
6764
6765
6766
6767
6768
6769
6770
6771
6772
6773
6774
6775
6776
6777
6778
6779
6780
6781
6782
6783
6784
6785
6786
6787
6788
6789
6790
6791
6792
6793
6794
6795
6796
6797
6798
6799
6800
6801
6802
6803
6804
6805
6806
6807
6808
6809
6810
6811
6812
6813
6814
6815
6816
6817
6818
6819
6820
6821
6822
6823
6824
6825
6826
6827
6828
6829
6830
6831
6832
6833
6834
6835
6836
6837
6838
6839
6840
6841
6842
6843
6844
6845
6846
6847
6848
6849
6850
6851
6852
6853
6854
6855
6856
6857
6858
6859
6860
6861
6862
6863
6864
6865
6866
6867
6868
6869
6870
6871
6872
6873
6874
6875
6876
6877
6878
6879
6880
6881
6882
6883
6884
6885
6886
6887
6888
6889
6890
6891
6892
6893
6894
6895
6896
6897
6898
6899
6900
6901
6902
6903
6904
6905
6906
6907
6908
6909
6910
6911
6912
6913
6914
6915
6916
6917
6918
6919
6920
6921
6922
6923
6924
6925
6926
6927
6928
6929
6930
6931
6932
6933
6934
6935
6936
6937
6938
6939
6940
6941
6942
6943
6944
6945
6946
6947
6948
6949
6950
6951
6952
6953
6954
6955
6956
6957
6958
6959
6960
6961
6962
6963
6964
6965
6966
6967
6968
6969
6970
6971
6972
6973
6974
6975
6976
6977
6978
6979
6980
6981
6982
6983
6984
6985
6986
6987
6988
6989
6990
6991
6992
6993
6994
6995
6996
6997
6998
6999
7000
7001
7002
7003
7004
7005
7006
7007
7008
7009
7010
7011
7012
7013
7014
7015
7016
7017
7018
7019
7020
7021
7022
7023
7024
7025
7026
7027
7028
7029
7030
7031
7032
7033
7034
7035
7036
7037
7038
7039
7040
7041
7042
7043
7044
7045
7046
7047
7048
7049
7050
7051
7052
7053
7054
7055
7056
7057
7058
7059
7060
7061
7062
7063
7064
7065
7066
7067
7068
7069
7070
7071
7072
7073
7074
7075
7076
7077
7078
7079
7080
7081
7082
7083
7084
7085
7086
7087
7088
7089
7090
7091
7092
7093
7094
7095
7096
7097
7098
7099
7100
7101
7102
7103
7104
7105
7106
7107
7108
7109
7110
7111
7112
7113
7114
7115
7116
7117
7118
7119
7120
7121
7122
7123
7124
7125
7126
7127
7128
7129
7130
7131
7132
7133
7134
7135
7136
7137
7138
7139
7140
7141
7142
7143
7144
7145
7146
7147
7148
7149
7150
7151
7152
7153
7154
7155
7156
7157
7158
7159
7160
7161
7162
7163
7164
7165
7166
7167
7168
7169
7170
7171
7172
7173
7174
7175
7176
7177
7178
7179
7180
7181
7182
7183
7184
7185
7186
7187
7188
7189
7190
7191
7192
7193
7194
7195
7196
7197
7198
7199
7200
7201
7202
7203
7204
7205
7206
7207
7208
7209
7210
7211
7212
7213
7214
7215
7216
7217
7218
7219
7220
7221
7222
7223
7224
7225
7226
7227
7228
7229
7230
7231
7232
7233
7234
7235
7236
7237
7238
7239
7240
7241
7242
7243
7244
7245
7246
7247
7248
7249
7250
7251
7252
7253
7254
7255
7256
7257
7258
7259
7260
7261
7262
7263
7264
7265
7266
7267
7268
7269
7270
7271
7272
7273
7274
7275
7276
7277
7278
7279
7280
7281
7282
7283
7284
7285
7286
7287
7288
7289
7290
7291
7292
7293
7294
7295
7296
7297
7298
7299
7300
7301
7302
7303
7304
7305
7306
7307
7308
7309
7310
7311
7312
7313
7314
7315
7316
7317
7318
7319
7320
7321
7322
7323
7324
7325
7326
7327
7328
7329
7330
7331
7332
7333
7334
7335
7336
7337
7338
7339
7340
7341
7342
7343
7344
7345
7346
7347
7348
7349
7350
7351
7352
7353
7354
7355
7356
7357
7358
7359
7360
7361
7362
7363
7364
7365
7366
7367
7368
7369
7370
7371
7372
7373
7374
7375
7376
7377
7378
7379
7380
7381
7382
7383
7384
7385
7386
7387
7388
7389
7390
7391
7392
7393
7394
7395
7396
7397
7398
7399
7400
7401
7402
7403
7404
7405
7406
7407
7408
7409
7410
7411
7412
7413
7414
7415
7416
7417
7418
7419
7420
7421
7422
7423
7424
7425
7426
7427
7428
7429
7430
7431
7432
7433
7434
7435
7436
7437
7438
7439
7440
7441
7442
7443
7444
7445
7446
7447
7448
7449
7450
7451
7452
7453
7454
7455
7456
7457
7458
7459
7460
7461
7462
7463
7464
7465
7466
7467
7468
7469
7470
7471
7472
7473
7474
7475
7476
7477
7478
7479
7480
7481
7482
7483
7484
7485
7486
7487
7488
7489
7490
7491
7492
7493
7494
7495
7496
7497
7498
7499
7500
7501
7502
7503
7504
7505
7506
7507
7508
7509
7510
7511
7512
7513
7514
7515
7516
7517
7518
7519
7520
7521
7522
7523
7524
7525
7526
7527
7528
7529
7530
7531
7532
7533
7534
7535
7536
7537
7538
7539
7540
7541
7542
7543
7544
7545
7546
7547
7548
7549
7550
7551
7552
7553
7554
7555
7556
7557
7558
7559
7560
7561
7562
7563
7564
7565
7566
7567
7568
7569
7570
7571
7572
7573
7574
7575
7576
7577
7578
7579
7580
7581
7582
7583
7584
7585
7586
7587
7588
7589
7590
7591
7592
7593
7594
7595
7596
7597
7598
7599
7600
7601
7602
7603
7604
7605
7606
7607
7608
7609
7610
7611
7612
7613
7614
7615
7616
7617
7618
7619
7620
7621
7622
7623
7624
7625
7626
7627
7628
7629
7630
7631
7632
7633
7634
7635
7636
7637
7638
7639
7640
7641
7642
7643
7644
7645
7646
7647
7648
7649
7650
7651
7652
7653
7654
7655
7656
7657
7658
7659
7660
7661
7662
7663
7664
7665
7666
7667
7668
7669
7670
7671
7672
7673
7674
7675
7676
7677
7678
7679
7680
7681
7682
7683
7684
7685
7686
7687
7688
7689
7690
7691
7692
7693
// SPDX-License-Identifier: GPL-2.0
/*
 * Generic ring buffer
 *
 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
 */
#include <linux/trace_recursion.h>
#include <linux/trace_events.h>
#include <linux/ring_buffer.h>
#include <linux/trace_clock.h>
#include <linux/sched/clock.h>
#include <linux/cacheflush.h>
#include <linux/trace_seq.h>
#include <linux/spinlock.h>
#include <linux/irq_work.h>
#include <linux/security.h>
#include <linux/uaccess.h>
#include <linux/hardirq.h>
#include <linux/kthread.h>        /* for self test */
#include <linux/module.h>
#include <linux/percpu.h>
#include <linux/mutex.h>
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/hash.h>
#include <linux/list.h>
#include <linux/cpu.h>
#include <linux/oom.h>
#include <linux/mm.h>

#include <asm/local64.h>
#include <asm/local.h>
#include <asm/setup.h>

#include "trace.h"

/*
 * The "absolute" timestamp in the buffer is only 59 bits.
 * If a clock has the 5 MSBs set, it needs to be saved and
 * reinserted.
 */
#define TS_MSB                (0xf8ULL << 56)
#define ABS_TS_MASK        (~TS_MSB)

static void update_pages_handler(struct work_struct *work);

#define RING_BUFFER_META_MAGIC        0xBADFEED

struct ring_buffer_meta {
        int                magic;
        int                struct_sizes;
        unsigned long        total_size;
        unsigned long        buffers_offset;
};

struct ring_buffer_cpu_meta {
        unsigned long        first_buffer;
        unsigned long        head_buffer;
        unsigned long        commit_buffer;
        __u32                subbuf_size;
        __u32                nr_subbufs;
        int                buffers[];
};

/*
 * The ring buffer header is special. We must manually up keep it.
 */
int ring_buffer_print_entry_header(struct trace_seq *s)
{
        trace_seq_puts(s, "# compressed entry header\n");
        trace_seq_puts(s, "\ttype_len    :    5 bits\n");
        trace_seq_puts(s, "\ttime_delta  :   27 bits\n");
        trace_seq_puts(s, "\tarray       :   32 bits\n");
        trace_seq_putc(s, '\n');
        trace_seq_printf(s, "\tpadding     : type == %d\n",
                         RINGBUF_TYPE_PADDING);
        trace_seq_printf(s, "\ttime_extend : type == %d\n",
                         RINGBUF_TYPE_TIME_EXTEND);
        trace_seq_printf(s, "\ttime_stamp : type == %d\n",
                         RINGBUF_TYPE_TIME_STAMP);
        trace_seq_printf(s, "\tdata max type_len  == %d\n",
                         RINGBUF_TYPE_DATA_TYPE_LEN_MAX);

        return !trace_seq_has_overflowed(s);
}

/*
 * The ring buffer is made up of a list of pages. A separate list of pages is
 * allocated for each CPU. A writer may only write to a buffer that is
 * associated with the CPU it is currently executing on.  A reader may read
 * from any per cpu buffer.
 *
 * The reader is special. For each per cpu buffer, the reader has its own
 * reader page. When a reader has read the entire reader page, this reader
 * page is swapped with another page in the ring buffer.
 *
 * Now, as long as the writer is off the reader page, the reader can do what
 * ever it wants with that page. The writer will never write to that page
 * again (as long as it is out of the ring buffer).
 *
 * Here's some silly ASCII art.
 *
 *   +------+
 *   |reader|          RING BUFFER
 *   |page  |
 *   +------+        +---+   +---+   +---+
 *                   |   |-->|   |-->|   |
 *                   +---+   +---+   +---+
 *                     ^               |
 *                     |               |
 *                     +---------------+
 *
 *
 *   +------+
 *   |reader|          RING BUFFER
 *   |page  |------------------v
 *   +------+        +---+   +---+   +---+
 *                   |   |-->|   |-->|   |
 *                   +---+   +---+   +---+
 *                     ^               |
 *                     |               |
 *                     +---------------+
 *
 *
 *   +------+
 *   |reader|          RING BUFFER
 *   |page  |------------------v
 *   +------+        +---+   +---+   +---+
 *      ^            |   |-->|   |-->|   |
 *      |            +---+   +---+   +---+
 *      |                              |
 *      |                              |
 *      +------------------------------+
 *
 *
 *   +------+
 *   |buffer|          RING BUFFER
 *   |page  |------------------v
 *   +------+        +---+   +---+   +---+
 *      ^            |   |   |   |-->|   |
 *      |   New      +---+   +---+   +---+
 *      |  Reader------^               |
 *      |   page                       |
 *      +------------------------------+
 *
 *
 * After we make this swap, the reader can hand this page off to the splice
 * code and be done with it. It can even allocate a new page if it needs to
 * and swap that into the ring buffer.
 *
 * We will be using cmpxchg soon to make all this lockless.
 *
 */

/* Used for individual buffers (after the counter) */
#define RB_BUFFER_OFF                (1 << 20)

#define BUF_PAGE_HDR_SIZE offsetof(struct buffer_data_page, data)

#define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array))
#define RB_ALIGNMENT                4U
#define RB_MAX_SMALL_DATA        (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
#define RB_EVNT_MIN_SIZE        8U        /* two 32bit words */

#ifndef CONFIG_HAVE_64BIT_ALIGNED_ACCESS
# define RB_FORCE_8BYTE_ALIGNMENT        0
# define RB_ARCH_ALIGNMENT                RB_ALIGNMENT
#else
# define RB_FORCE_8BYTE_ALIGNMENT        1
# define RB_ARCH_ALIGNMENT                8U
#endif

#define RB_ALIGN_DATA                __aligned(RB_ARCH_ALIGNMENT)

/* define RINGBUF_TYPE_DATA for 'case RINGBUF_TYPE_DATA:' */
#define RINGBUF_TYPE_DATA 0 ... RINGBUF_TYPE_DATA_TYPE_LEN_MAX

enum {
        RB_LEN_TIME_EXTEND = 8,
        RB_LEN_TIME_STAMP =  8,
};

#define skip_time_extend(event) \
        ((struct ring_buffer_event *)((char *)event + RB_LEN_TIME_EXTEND))

#define extended_time(event) \
        (event->type_len >= RINGBUF_TYPE_TIME_EXTEND)

static inline bool rb_null_event(struct ring_buffer_event *event)
{
        return event->type_len == RINGBUF_TYPE_PADDING && !event->time_delta;
}

static void rb_event_set_padding(struct ring_buffer_event *event)
{
        /* padding has a NULL time_delta */
        event->type_len = RINGBUF_TYPE_PADDING;
        event->time_delta = 0;
}

static unsigned
rb_event_data_length(struct ring_buffer_event *event)
{
        unsigned length;

        if (event->type_len)
                length = event->type_len * RB_ALIGNMENT;
        else
                length = event->array[0];
        return length + RB_EVNT_HDR_SIZE;
}

/*
 * Return the length of the given event. Will return
 * the length of the time extend if the event is a
 * time extend.
 */
static inline unsigned
rb_event_length(struct ring_buffer_event *event)
{
        switch (event->type_len) {
        case RINGBUF_TYPE_PADDING:
                if (rb_null_event(event))
                        /* undefined */
                        return -1;
                return  event->array[0] + RB_EVNT_HDR_SIZE;

        case RINGBUF_TYPE_TIME_EXTEND:
                return RB_LEN_TIME_EXTEND;

        case RINGBUF_TYPE_TIME_STAMP:
                return RB_LEN_TIME_STAMP;

        case RINGBUF_TYPE_DATA:
                return rb_event_data_length(event);
        default:
                WARN_ON_ONCE(1);
        }
        /* not hit */
        return 0;
}

/*
 * Return total length of time extend and data,
 *   or just the event length for all other events.
 */
static inline unsigned
rb_event_ts_length(struct ring_buffer_event *event)
{
        unsigned len = 0;

        if (extended_time(event)) {
                /* time extends include the data event after it */
                len = RB_LEN_TIME_EXTEND;
                event = skip_time_extend(event);
        }
        return len + rb_event_length(event);
}

/**
 * ring_buffer_event_length - return the length of the event
 * @event: the event to get the length of
 *
 * Returns the size of the data load of a data event.
 * If the event is something other than a data event, it
 * returns the size of the event itself. With the exception
 * of a TIME EXTEND, where it still returns the size of the
 * data load of the data event after it.
 */
unsigned ring_buffer_event_length(struct ring_buffer_event *event)
{
        unsigned length;

        if (extended_time(event))
                event = skip_time_extend(event);

        length = rb_event_length(event);
        if (event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
                return length;
        length -= RB_EVNT_HDR_SIZE;
        if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0]))
                length -= sizeof(event->array[0]);
        return length;
}
EXPORT_SYMBOL_GPL(ring_buffer_event_length);

/* inline for ring buffer fast paths */
static __always_inline void *
rb_event_data(struct ring_buffer_event *event)
{
        if (extended_time(event))
                event = skip_time_extend(event);
        WARN_ON_ONCE(event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX);
        /* If length is in len field, then array[0] has the data */
        if (event->type_len)
                return (void *)&event->array[0];
        /* Otherwise length is in array[0] and array[1] has the data */
        return (void *)&event->array[1];
}

/**
 * ring_buffer_event_data - return the data of the event
 * @event: the event to get the data from
 */
void *ring_buffer_event_data(struct ring_buffer_event *event)
{
        return rb_event_data(event);
}
EXPORT_SYMBOL_GPL(ring_buffer_event_data);

#define for_each_buffer_cpu(buffer, cpu)                \
        for_each_cpu(cpu, buffer->cpumask)

#define for_each_online_buffer_cpu(buffer, cpu)                \
        for_each_cpu_and(cpu, buffer->cpumask, cpu_online_mask)

#define TS_SHIFT        27
#define TS_MASK                ((1ULL << TS_SHIFT) - 1)
#define TS_DELTA_TEST        (~TS_MASK)

static u64 rb_event_time_stamp(struct ring_buffer_event *event)
{
        u64 ts;

        ts = event->array[0];
        ts <<= TS_SHIFT;
        ts += event->time_delta;

        return ts;
}

/* Flag when events were overwritten */
#define RB_MISSED_EVENTS        (1 << 31)
/* Missed count stored at end */
#define RB_MISSED_STORED        (1 << 30)

#define RB_MISSED_MASK                (3 << 30)

struct buffer_data_page {
        u64                 time_stamp;        /* page time stamp */
        local_t                 commit;        /* write committed index */
        unsigned char         data[] RB_ALIGN_DATA;        /* data of buffer page */
};

struct buffer_data_read_page {
        unsigned                order;        /* order of the page */
        struct buffer_data_page        *data;        /* actual data, stored in this page */
};

/*
 * Note, the buffer_page list must be first. The buffer pages
 * are allocated in cache lines, which means that each buffer
 * page will be at the beginning of a cache line, and thus
 * the least significant bits will be zero. We use this to
 * add flags in the list struct pointers, to make the ring buffer
 * lockless.
 */
struct buffer_page {
        struct list_head list;                /* list of buffer pages */
        local_t                 write;                /* index for next write */
        unsigned         read;                /* index for next read */
        local_t                 entries;        /* entries on this page */
        unsigned long         real_end;        /* real end of data */
        unsigned         order;                /* order of the page */
        u32                 id:30;                /* ID for external mapping */
        u32                 range:1;        /* Mapped via a range */
        struct buffer_data_page *page;        /* Actual data page */
};

/*
 * The buffer page counters, write and entries, must be reset
 * atomically when crossing page boundaries. To synchronize this
 * update, two counters are inserted into the number. One is
 * the actual counter for the write position or count on the page.
 *
 * The other is a counter of updaters. Before an update happens
 * the update partition of the counter is incremented. This will
 * allow the updater to update the counter atomically.
 *
 * The counter is 20 bits, and the state data is 12.
 */
#define RB_WRITE_MASK                0xfffff
#define RB_WRITE_INTCNT                (1 << 20)

static void rb_init_page(struct buffer_data_page *bpage)
{
        local_set(&bpage->commit, 0);
}

static __always_inline unsigned int rb_page_commit(struct buffer_page *bpage)
{
        return local_read(&bpage->page->commit);
}

static void free_buffer_page(struct buffer_page *bpage)
{
        /* Range pages are not to be freed */
        if (!bpage->range)
                free_pages((unsigned long)bpage->page, bpage->order);
        kfree(bpage);
}

/*
 * We need to fit the time_stamp delta into 27 bits.
 */
static inline bool test_time_stamp(u64 delta)
{
        return !!(delta & TS_DELTA_TEST);
}

struct rb_irq_work {
        struct irq_work                        work;
        wait_queue_head_t                waiters;
        wait_queue_head_t                full_waiters;
        atomic_t                        seq;
        bool                                waiters_pending;
        bool                                full_waiters_pending;
        bool                                wakeup_full;
};

/*
 * Structure to hold event state and handle nested events.
 */
struct rb_event_info {
        u64                        ts;
        u64                        delta;
        u64                        before;
        u64                        after;
        unsigned long                length;
        struct buffer_page        *tail_page;
        int                        add_timestamp;
};

/*
 * Used for the add_timestamp
 *  NONE
 *  EXTEND - wants a time extend
 *  ABSOLUTE - the buffer requests all events to have absolute time stamps
 *  FORCE - force a full time stamp.
 */
enum {
        RB_ADD_STAMP_NONE                = 0,
        RB_ADD_STAMP_EXTEND                = BIT(1),
        RB_ADD_STAMP_ABSOLUTE                = BIT(2),
        RB_ADD_STAMP_FORCE                = BIT(3)
};
/*
 * Used for which event context the event is in.
 *  TRANSITION = 0
 *  NMI     = 1
 *  IRQ     = 2
 *  SOFTIRQ = 3
 *  NORMAL  = 4
 *
 * See trace_recursive_lock() comment below for more details.
 */
enum {
        RB_CTX_TRANSITION,
        RB_CTX_NMI,
        RB_CTX_IRQ,
        RB_CTX_SOFTIRQ,
        RB_CTX_NORMAL,
        RB_CTX_MAX
};

struct rb_time_struct {
        local64_t        time;
};
typedef struct rb_time_struct rb_time_t;

#define MAX_NEST        5

/*
 * head_page == tail_page && head == tail then buffer is empty.
 */
struct ring_buffer_per_cpu {
        int                                cpu;
        atomic_t                        record_disabled;
        atomic_t                        resize_disabled;
        struct trace_buffer        *buffer;
        raw_spinlock_t                        reader_lock;        /* serialize readers */
        arch_spinlock_t                        lock;
        struct lock_class_key                lock_key;
        struct buffer_data_page                *free_page;
        unsigned long                        nr_pages;
        unsigned int                        current_context;
        struct list_head                *pages;
        /* pages generation counter, incremented when the list changes */
        unsigned long                        cnt;
        struct buffer_page                *head_page;        /* read from head */
        struct buffer_page                *tail_page;        /* write to tail */
        struct buffer_page                *commit_page;        /* committed pages */
        struct buffer_page                *reader_page;
        unsigned long                        lost_events;
        unsigned long                        last_overrun;
        unsigned long                        nest;
        local_t                                entries_bytes;
        local_t                                entries;
        local_t                                overrun;
        local_t                                commit_overrun;
        local_t                                dropped_events;
        local_t                                committing;
        local_t                                commits;
        local_t                                pages_touched;
        local_t                                pages_lost;
        local_t                                pages_read;
        long                                last_pages_touch;
        size_t                                shortest_full;
        unsigned long                        read;
        unsigned long                        read_bytes;
        rb_time_t                        write_stamp;
        rb_time_t                        before_stamp;
        u64                                event_stamp[MAX_NEST];
        u64                                read_stamp;
        /* pages removed since last reset */
        unsigned long                        pages_removed;

        unsigned int                        mapped;
        unsigned int                        user_mapped;        /* user space mapping */
        struct mutex                        mapping_lock;
        unsigned long                        *subbuf_ids;        /* ID to subbuf VA */
        struct trace_buffer_meta        *meta_page;
        struct ring_buffer_cpu_meta        *ring_meta;

        /* ring buffer pages to update, > 0 to add, < 0 to remove */
        long                                nr_pages_to_update;
        struct list_head                new_pages; /* new pages to add */
        struct work_struct                update_pages_work;
        struct completion                update_done;

        struct rb_irq_work                irq_work;
};

struct trace_buffer {
        unsigned                        flags;
        int                                cpus;
        atomic_t                        record_disabled;
        atomic_t                        resizing;
        cpumask_var_t                        cpumask;

        struct lock_class_key                *reader_lock_key;

        struct mutex                        mutex;

        struct ring_buffer_per_cpu        **buffers;

        struct hlist_node                node;
        u64                                (*clock)(void);

        struct rb_irq_work                irq_work;
        bool                                time_stamp_abs;

        unsigned long                        range_addr_start;
        unsigned long                        range_addr_end;

        struct ring_buffer_meta                *meta;

        unsigned int                        subbuf_size;
        unsigned int                        subbuf_order;
        unsigned int                        max_data_size;
};

struct ring_buffer_iter {
        struct ring_buffer_per_cpu        *cpu_buffer;
        unsigned long                        head;
        unsigned long                        next_event;
        struct buffer_page                *head_page;
        struct buffer_page                *cache_reader_page;
        unsigned long                        cache_read;
        unsigned long                        cache_pages_removed;
        u64                                read_stamp;
        u64                                page_stamp;
        struct ring_buffer_event        *event;
        size_t                                event_size;
        int                                missed_events;
};

int ring_buffer_print_page_header(struct trace_buffer *buffer, struct trace_seq *s)
{
        struct buffer_data_page field;

        trace_seq_printf(s, "\tfield: u64 timestamp;\t"
                         "offset:0;\tsize:%u;\tsigned:%u;\n",
                         (unsigned int)sizeof(field.time_stamp),
                         (unsigned int)is_signed_type(u64));

        trace_seq_printf(s, "\tfield: local_t commit;\t"
                         "offset:%u;\tsize:%u;\tsigned:%u;\n",
                         (unsigned int)offsetof(typeof(field), commit),
                         (unsigned int)sizeof(field.commit),
                         (unsigned int)is_signed_type(long));

        trace_seq_printf(s, "\tfield: int overwrite;\t"
                         "offset:%u;\tsize:%u;\tsigned:%u;\n",
                         (unsigned int)offsetof(typeof(field), commit),
                         1,
                         (unsigned int)is_signed_type(long));

        trace_seq_printf(s, "\tfield: char data;\t"
                         "offset:%u;\tsize:%u;\tsigned:%u;\n",
                         (unsigned int)offsetof(typeof(field), data),
                         (unsigned int)buffer->subbuf_size,
                         (unsigned int)is_signed_type(char));

        return !trace_seq_has_overflowed(s);
}

static inline void rb_time_read(rb_time_t *t, u64 *ret)
{
        *ret = local64_read(&t->time);
}
static void rb_time_set(rb_time_t *t, u64 val)
{
        local64_set(&t->time, val);
}

/*
 * Enable this to make sure that the event passed to
 * ring_buffer_event_time_stamp() is not committed and also
 * is on the buffer that it passed in.
 */
//#define RB_VERIFY_EVENT
#ifdef RB_VERIFY_EVENT
static struct list_head *rb_list_head(struct list_head *list);
static void verify_event(struct ring_buffer_per_cpu *cpu_buffer,
                         void *event)
{
        struct buffer_page *page = cpu_buffer->commit_page;
        struct buffer_page *tail_page = READ_ONCE(cpu_buffer->tail_page);
        struct list_head *next;
        long commit, write;
        unsigned long addr = (unsigned long)event;
        bool done = false;
        int stop = 0;

        /* Make sure the event exists and is not committed yet */
        do {
                if (page == tail_page || WARN_ON_ONCE(stop++ > 100))
                        done = true;
                commit = local_read(&page->page->commit);
                write = local_read(&page->write);
                if (addr >= (unsigned long)&page->page->data[commit] &&
                    addr < (unsigned long)&page->page->data[write])
                        return;

                next = rb_list_head(page->list.next);
                page = list_entry(next, struct buffer_page, list);
        } while (!done);
        WARN_ON_ONCE(1);
}
#else
static inline void verify_event(struct ring_buffer_per_cpu *cpu_buffer,
                         void *event)
{
}
#endif

/*
 * The absolute time stamp drops the 5 MSBs and some clocks may
 * require them. The rb_fix_abs_ts() will take a previous full
 * time stamp, and add the 5 MSB of that time stamp on to the
 * saved absolute time stamp. Then they are compared in case of
 * the unlikely event that the latest time stamp incremented
 * the 5 MSB.
 */
static inline u64 rb_fix_abs_ts(u64 abs, u64 save_ts)
{
        if (save_ts & TS_MSB) {
                abs |= save_ts & TS_MSB;
                /* Check for overflow */
                if (unlikely(abs < save_ts))
                        abs += 1ULL << 59;
        }
        return abs;
}

static inline u64 rb_time_stamp(struct trace_buffer *buffer);

/**
 * ring_buffer_event_time_stamp - return the event's current time stamp
 * @buffer: The buffer that the event is on
 * @event: the event to get the time stamp of
 *
 * Note, this must be called after @event is reserved, and before it is
 * committed to the ring buffer. And must be called from the same
 * context where the event was reserved (normal, softirq, irq, etc).
 *
 * Returns the time stamp associated with the current event.
 * If the event has an extended time stamp, then that is used as
 * the time stamp to return.
 * In the highly unlikely case that the event was nested more than
 * the max nesting, then the write_stamp of the buffer is returned,
 * otherwise  current time is returned, but that really neither of
 * the last two cases should ever happen.
 */
u64 ring_buffer_event_time_stamp(struct trace_buffer *buffer,
                                 struct ring_buffer_event *event)
{
        struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[smp_processor_id()];
        unsigned int nest;
        u64 ts;

        /* If the event includes an absolute time, then just use that */
        if (event->type_len == RINGBUF_TYPE_TIME_STAMP) {
                ts = rb_event_time_stamp(event);
                return rb_fix_abs_ts(ts, cpu_buffer->tail_page->page->time_stamp);
        }

        nest = local_read(&cpu_buffer->committing);
        verify_event(cpu_buffer, event);
        if (WARN_ON_ONCE(!nest))
                goto fail;

        /* Read the current saved nesting level time stamp */
        if (likely(--nest < MAX_NEST))
                return cpu_buffer->event_stamp[nest];

        /* Shouldn't happen, warn if it does */
        WARN_ONCE(1, "nest (%d) greater than max", nest);

 fail:
        rb_time_read(&cpu_buffer->write_stamp, &ts);

        return ts;
}

/**
 * ring_buffer_nr_dirty_pages - get the number of used pages in the ring buffer
 * @buffer: The ring_buffer to get the number of pages from
 * @cpu: The cpu of the ring_buffer to get the number of pages from
 *
 * Returns the number of pages that have content in the ring buffer.
 */
size_t ring_buffer_nr_dirty_pages(struct trace_buffer *buffer, int cpu)
{
        size_t read;
        size_t lost;
        size_t cnt;

        read = local_read(&buffer->buffers[cpu]->pages_read);
        lost = local_read(&buffer->buffers[cpu]->pages_lost);
        cnt = local_read(&buffer->buffers[cpu]->pages_touched);

        if (WARN_ON_ONCE(cnt < lost))
                return 0;

        cnt -= lost;

        /* The reader can read an empty page, but not more than that */
        if (cnt < read) {
                WARN_ON_ONCE(read > cnt + 1);
                return 0;
        }

        return cnt - read;
}

static __always_inline bool full_hit(struct trace_buffer *buffer, int cpu, int full)
{
        struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
        size_t nr_pages;
        size_t dirty;

        nr_pages = cpu_buffer->nr_pages;
        if (!nr_pages || !full)
                return true;

        /*
         * Add one as dirty will never equal nr_pages, as the sub-buffer
         * that the writer is on is not counted as dirty.
         * This is needed if "buffer_percent" is set to 100.
         */
        dirty = ring_buffer_nr_dirty_pages(buffer, cpu) + 1;

        return (dirty * 100) >= (full * nr_pages);
}

/*
 * rb_wake_up_waiters - wake up tasks waiting for ring buffer input
 *
 * Schedules a delayed work to wake up any task that is blocked on the
 * ring buffer waiters queue.
 */
static void rb_wake_up_waiters(struct irq_work *work)
{
        struct rb_irq_work *rbwork = container_of(work, struct rb_irq_work, work);

        /* For waiters waiting for the first wake up */
        (void)atomic_fetch_inc_release(&rbwork->seq);

        wake_up_all(&rbwork->waiters);
        if (rbwork->full_waiters_pending || rbwork->wakeup_full) {
                /* Only cpu_buffer sets the above flags */
                struct ring_buffer_per_cpu *cpu_buffer =
                        container_of(rbwork, struct ring_buffer_per_cpu, irq_work);

                /* Called from interrupt context */
                raw_spin_lock(&cpu_buffer->reader_lock);
                rbwork->wakeup_full = false;
                rbwork->full_waiters_pending = false;

                /* Waking up all waiters, they will reset the shortest full */
                cpu_buffer->shortest_full = 0;
                raw_spin_unlock(&cpu_buffer->reader_lock);

                wake_up_all(&rbwork->full_waiters);
        }
}

/**
 * ring_buffer_wake_waiters - wake up any waiters on this ring buffer
 * @buffer: The ring buffer to wake waiters on
 * @cpu: The CPU buffer to wake waiters on
 *
 * In the case of a file that represents a ring buffer is closing,
 * it is prudent to wake up any waiters that are on this.
 */
void ring_buffer_wake_waiters(struct trace_buffer *buffer, int cpu)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        struct rb_irq_work *rbwork;

        if (!buffer)
                return;

        if (cpu == RING_BUFFER_ALL_CPUS) {

                /* Wake up individual ones too. One level recursion */
                for_each_buffer_cpu(buffer, cpu)
                        ring_buffer_wake_waiters(buffer, cpu);

                rbwork = &buffer->irq_work;
        } else {
                if (WARN_ON_ONCE(!buffer->buffers))
                        return;
                if (WARN_ON_ONCE(cpu >= nr_cpu_ids))
                        return;

                cpu_buffer = buffer->buffers[cpu];
                /* The CPU buffer may not have been initialized yet */
                if (!cpu_buffer)
                        return;
                rbwork = &cpu_buffer->irq_work;
        }

        /* This can be called in any context */
        irq_work_queue(&rbwork->work);
}

static bool rb_watermark_hit(struct trace_buffer *buffer, int cpu, int full)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        bool ret = false;

        /* Reads of all CPUs always waits for any data */
        if (cpu == RING_BUFFER_ALL_CPUS)
                return !ring_buffer_empty(buffer);

        cpu_buffer = buffer->buffers[cpu];

        if (!ring_buffer_empty_cpu(buffer, cpu)) {
                unsigned long flags;
                bool pagebusy;

                if (!full)
                        return true;

                raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
                pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page;
                ret = !pagebusy && full_hit(buffer, cpu, full);

                if (!ret && (!cpu_buffer->shortest_full ||
                             cpu_buffer->shortest_full > full)) {
                    cpu_buffer->shortest_full = full;
                }
                raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
        }
        return ret;
}

static inline bool
rb_wait_cond(struct rb_irq_work *rbwork, struct trace_buffer *buffer,
             int cpu, int full, ring_buffer_cond_fn cond, void *data)
{
        if (rb_watermark_hit(buffer, cpu, full))
                return true;

        if (cond(data))
                return true;

        /*
         * The events can happen in critical sections where
         * checking a work queue can cause deadlocks.
         * After adding a task to the queue, this flag is set
         * only to notify events to try to wake up the queue
         * using irq_work.
         *
         * We don't clear it even if the buffer is no longer
         * empty. The flag only causes the next event to run
         * irq_work to do the work queue wake up. The worse
         * that can happen if we race with !trace_empty() is that
         * an event will cause an irq_work to try to wake up
         * an empty queue.
         *
         * There's no reason to protect this flag either, as
         * the work queue and irq_work logic will do the necessary
         * synchronization for the wake ups. The only thing
         * that is necessary is that the wake up happens after
         * a task has been queued. It's OK for spurious wake ups.
         */
        if (full)
                rbwork->full_waiters_pending = true;
        else
                rbwork->waiters_pending = true;

        return false;
}

struct rb_wait_data {
        struct rb_irq_work                *irq_work;
        int                                seq;
};

/*
 * The default wait condition for ring_buffer_wait() is to just to exit the
 * wait loop the first time it is woken up.
 */
static bool rb_wait_once(void *data)
{
        struct rb_wait_data *rdata = data;
        struct rb_irq_work *rbwork = rdata->irq_work;

        return atomic_read_acquire(&rbwork->seq) != rdata->seq;
}

/**
 * ring_buffer_wait - wait for input to the ring buffer
 * @buffer: buffer to wait on
 * @cpu: the cpu buffer to wait on
 * @full: wait until the percentage of pages are available, if @cpu != RING_BUFFER_ALL_CPUS
 * @cond: condition function to break out of wait (NULL to run once)
 * @data: the data to pass to @cond.
 *
 * If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon
 * as data is added to any of the @buffer's cpu buffers. Otherwise
 * it will wait for data to be added to a specific cpu buffer.
 */
int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full,
                     ring_buffer_cond_fn cond, void *data)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        struct wait_queue_head *waitq;
        struct rb_irq_work *rbwork;
        struct rb_wait_data rdata;
        int ret = 0;

        /*
         * Depending on what the caller is waiting for, either any
         * data in any cpu buffer, or a specific buffer, put the
         * caller on the appropriate wait queue.
         */
        if (cpu == RING_BUFFER_ALL_CPUS) {
                rbwork = &buffer->irq_work;
                /* Full only makes sense on per cpu reads */
                full = 0;
        } else {
                if (!cpumask_test_cpu(cpu, buffer->cpumask))
                        return -ENODEV;
                cpu_buffer = buffer->buffers[cpu];
                rbwork = &cpu_buffer->irq_work;
        }

        if (full)
                waitq = &rbwork->full_waiters;
        else
                waitq = &rbwork->waiters;

        /* Set up to exit loop as soon as it is woken */
        if (!cond) {
                cond = rb_wait_once;
                rdata.irq_work = rbwork;
                rdata.seq = atomic_read_acquire(&rbwork->seq);
                data = &rdata;
        }

        ret = wait_event_interruptible((*waitq),
                                rb_wait_cond(rbwork, buffer, cpu, full, cond, data));

        return ret;
}

/**
 * ring_buffer_poll_wait - poll on buffer input
 * @buffer: buffer to wait on
 * @cpu: the cpu buffer to wait on
 * @filp: the file descriptor
 * @poll_table: The poll descriptor
 * @full: wait until the percentage of pages are available, if @cpu != RING_BUFFER_ALL_CPUS
 *
 * If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon
 * as data is added to any of the @buffer's cpu buffers. Otherwise
 * it will wait for data to be added to a specific cpu buffer.
 *
 * Returns EPOLLIN | EPOLLRDNORM if data exists in the buffers,
 * zero otherwise.
 */
__poll_t ring_buffer_poll_wait(struct trace_buffer *buffer, int cpu,
                          struct file *filp, poll_table *poll_table, int full)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        struct rb_irq_work *rbwork;

        if (cpu == RING_BUFFER_ALL_CPUS) {
                rbwork = &buffer->irq_work;
                full = 0;
        } else {
                if (!cpumask_test_cpu(cpu, buffer->cpumask))
                        return EPOLLERR;

                cpu_buffer = buffer->buffers[cpu];
                rbwork = &cpu_buffer->irq_work;
        }

        if (full) {
                poll_wait(filp, &rbwork->full_waiters, poll_table);

                if (rb_watermark_hit(buffer, cpu, full))
                        return EPOLLIN | EPOLLRDNORM;
                /*
                 * Only allow full_waiters_pending update to be seen after
                 * the shortest_full is set (in rb_watermark_hit). If the
                 * writer sees the full_waiters_pending flag set, it will
                 * compare the amount in the ring buffer to shortest_full.
                 * If the amount in the ring buffer is greater than the
                 * shortest_full percent, it will call the irq_work handler
                 * to wake up this list. The irq_handler will reset shortest_full
                 * back to zero. That's done under the reader_lock, but
                 * the below smp_mb() makes sure that the update to
                 * full_waiters_pending doesn't leak up into the above.
                 */
                smp_mb();
                rbwork->full_waiters_pending = true;
                return 0;
        }

        poll_wait(filp, &rbwork->waiters, poll_table);
        rbwork->waiters_pending = true;

        /*
         * There's a tight race between setting the waiters_pending and
         * checking if the ring buffer is empty.  Once the waiters_pending bit
         * is set, the next event will wake the task up, but we can get stuck
         * if there's only a single event in.
         *
         * FIXME: Ideally, we need a memory barrier on the writer side as well,
         * but adding a memory barrier to all events will cause too much of a
         * performance hit in the fast path.  We only need a memory barrier when
         * the buffer goes from empty to having content.  But as this race is
         * extremely small, and it's not a problem if another event comes in, we
         * will fix it later.
         */
        smp_mb();

        if ((cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer)) ||
            (cpu != RING_BUFFER_ALL_CPUS && !ring_buffer_empty_cpu(buffer, cpu)))
                return EPOLLIN | EPOLLRDNORM;
        return 0;
}

/* buffer may be either ring_buffer or ring_buffer_per_cpu */
#define RB_WARN_ON(b, cond)                                                \
        ({                                                                \
                int _____ret = unlikely(cond);                                \
                if (_____ret) {                                                \
                        if (__same_type(*(b), struct ring_buffer_per_cpu)) { \
                                struct ring_buffer_per_cpu *__b =        \
                                        (void *)b;                        \
                                atomic_inc(&__b->buffer->record_disabled); \
                        } else                                                \
                                atomic_inc(&b->record_disabled);        \
                        WARN_ON(1);                                        \
                }                                                        \
                _____ret;                                                \
        })

/* Up this if you want to test the TIME_EXTENTS and normalization */
#define DEBUG_SHIFT 0

static inline u64 rb_time_stamp(struct trace_buffer *buffer)
{
        u64 ts;

        /* Skip retpolines :-( */
        if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) && likely(buffer->clock == trace_clock_local))
                ts = trace_clock_local();
        else
                ts = buffer->clock();

        /* shift to debug/test normalization and TIME_EXTENTS */
        return ts << DEBUG_SHIFT;
}

u64 ring_buffer_time_stamp(struct trace_buffer *buffer)
{
        u64 time;

        preempt_disable_notrace();
        time = rb_time_stamp(buffer);
        preempt_enable_notrace();

        return time;
}
EXPORT_SYMBOL_GPL(ring_buffer_time_stamp);

void ring_buffer_normalize_time_stamp(struct trace_buffer *buffer,
                                      int cpu, u64 *ts)
{
        /* Just stupid testing the normalize function and deltas */
        *ts >>= DEBUG_SHIFT;
}
EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp);

/*
 * Making the ring buffer lockless makes things tricky.
 * Although writes only happen on the CPU that they are on,
 * and they only need to worry about interrupts. Reads can
 * happen on any CPU.
 *
 * The reader page is always off the ring buffer, but when the
 * reader finishes with a page, it needs to swap its page with
 * a new one from the buffer. The reader needs to take from
 * the head (writes go to the tail). But if a writer is in overwrite
 * mode and wraps, it must push the head page forward.
 *
 * Here lies the problem.
 *
 * The reader must be careful to replace only the head page, and
 * not another one. As described at the top of the file in the
 * ASCII art, the reader sets its old page to point to the next
 * page after head. It then sets the page after head to point to
 * the old reader page. But if the writer moves the head page
 * during this operation, the reader could end up with the tail.
 *
 * We use cmpxchg to help prevent this race. We also do something
 * special with the page before head. We set the LSB to 1.
 *
 * When the writer must push the page forward, it will clear the
 * bit that points to the head page, move the head, and then set
 * the bit that points to the new head page.
 *
 * We also don't want an interrupt coming in and moving the head
 * page on another writer. Thus we use the second LSB to catch
 * that too. Thus:
 *
 * head->list->prev->next        bit 1          bit 0
 *                              -------        -------
 * Normal page                     0              0
 * Points to head page             0              1
 * New head page                   1              0
 *
 * Note we can not trust the prev pointer of the head page, because:
 *
 * +----+       +-----+        +-----+
 * |    |------>|  T  |---X--->|  N  |
 * |    |<------|     |        |     |
 * +----+       +-----+        +-----+
 *   ^                           ^ |
 *   |          +-----+          | |
 *   +----------|  R  |----------+ |
 *              |     |<-----------+
 *              +-----+
 *
 * Key:  ---X-->  HEAD flag set in pointer
 *         T      Tail page
 *         R      Reader page
 *         N      Next page
 *
 * (see __rb_reserve_next() to see where this happens)
 *
 *  What the above shows is that the reader just swapped out
 *  the reader page with a page in the buffer, but before it
 *  could make the new header point back to the new page added
 *  it was preempted by a writer. The writer moved forward onto
 *  the new page added by the reader and is about to move forward
 *  again.
 *
 *  You can see, it is legitimate for the previous pointer of
 *  the head (or any page) not to point back to itself. But only
 *  temporarily.
 */

#define RB_PAGE_NORMAL                0UL
#define RB_PAGE_HEAD                1UL
#define RB_PAGE_UPDATE                2UL


#define RB_FLAG_MASK                3UL

/* PAGE_MOVED is not part of the mask */
#define RB_PAGE_MOVED                4UL

/*
 * rb_list_head - remove any bit
 */
static struct list_head *rb_list_head(struct list_head *list)
{
        unsigned long val = (unsigned long)list;

        return (struct list_head *)(val & ~RB_FLAG_MASK);
}

/*
 * rb_is_head_page - test if the given page is the head page
 *
 * Because the reader may move the head_page pointer, we can
 * not trust what the head page is (it may be pointing to
 * the reader page). But if the next page is a header page,
 * its flags will be non zero.
 */
static inline int
rb_is_head_page(struct buffer_page *page, struct list_head *list)
{
        unsigned long val;

        val = (unsigned long)list->next;

        if ((val & ~RB_FLAG_MASK) != (unsigned long)&page->list)
                return RB_PAGE_MOVED;

        return val & RB_FLAG_MASK;
}

/*
 * rb_is_reader_page
 *
 * The unique thing about the reader page, is that, if the
 * writer is ever on it, the previous pointer never points
 * back to the reader page.
 */
static bool rb_is_reader_page(struct buffer_page *page)
{
        struct list_head *list = page->list.prev;

        return rb_list_head(list->next) != &page->list;
}

/*
 * rb_set_list_to_head - set a list_head to be pointing to head.
 */
static void rb_set_list_to_head(struct list_head *list)
{
        unsigned long *ptr;

        ptr = (unsigned long *)&list->next;
        *ptr |= RB_PAGE_HEAD;
        *ptr &= ~RB_PAGE_UPDATE;
}

/*
 * rb_head_page_activate - sets up head page
 */
static void rb_head_page_activate(struct ring_buffer_per_cpu *cpu_buffer)
{
        struct buffer_page *head;

        head = cpu_buffer->head_page;
        if (!head)
                return;

        /*
         * Set the previous list pointer to have the HEAD flag.
         */
        rb_set_list_to_head(head->list.prev);

        if (cpu_buffer->ring_meta) {
                struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta;
                meta->head_buffer = (unsigned long)head->page;
        }
}

static void rb_list_head_clear(struct list_head *list)
{
        unsigned long *ptr = (unsigned long *)&list->next;

        *ptr &= ~RB_FLAG_MASK;
}

/*
 * rb_head_page_deactivate - clears head page ptr (for free list)
 */
static void
rb_head_page_deactivate(struct ring_buffer_per_cpu *cpu_buffer)
{
        struct list_head *hd;

        /* Go through the whole list and clear any pointers found. */
        rb_list_head_clear(cpu_buffer->pages);

        list_for_each(hd, cpu_buffer->pages)
                rb_list_head_clear(hd);
}

static int rb_head_page_set(struct ring_buffer_per_cpu *cpu_buffer,
                            struct buffer_page *head,
                            struct buffer_page *prev,
                            int old_flag, int new_flag)
{
        struct list_head *list;
        unsigned long val = (unsigned long)&head->list;
        unsigned long ret;

        list = &prev->list;

        val &= ~RB_FLAG_MASK;

        ret = cmpxchg((unsigned long *)&list->next,
                      val | old_flag, val | new_flag);

        /* check if the reader took the page */
        if ((ret & ~RB_FLAG_MASK) != val)
                return RB_PAGE_MOVED;

        return ret & RB_FLAG_MASK;
}

static int rb_head_page_set_update(struct ring_buffer_per_cpu *cpu_buffer,
                                   struct buffer_page *head,
                                   struct buffer_page *prev,
                                   int old_flag)
{
        return rb_head_page_set(cpu_buffer, head, prev,
                                old_flag, RB_PAGE_UPDATE);
}

static int rb_head_page_set_head(struct ring_buffer_per_cpu *cpu_buffer,
                                 struct buffer_page *head,
                                 struct buffer_page *prev,
                                 int old_flag)
{
        return rb_head_page_set(cpu_buffer, head, prev,
                                old_flag, RB_PAGE_HEAD);
}

static int rb_head_page_set_normal(struct ring_buffer_per_cpu *cpu_buffer,
                                   struct buffer_page *head,
                                   struct buffer_page *prev,
                                   int old_flag)
{
        return rb_head_page_set(cpu_buffer, head, prev,
                                old_flag, RB_PAGE_NORMAL);
}

static inline void rb_inc_page(struct buffer_page **bpage)
{
        struct list_head *p = rb_list_head((*bpage)->list.next);

        *bpage = list_entry(p, struct buffer_page, list);
}

static struct buffer_page *
rb_set_head_page(struct ring_buffer_per_cpu *cpu_buffer)
{
        struct buffer_page *head;
        struct buffer_page *page;
        struct list_head *list;
        int i;

        if (RB_WARN_ON(cpu_buffer, !cpu_buffer->head_page))
                return NULL;

        /* sanity check */
        list = cpu_buffer->pages;
        if (RB_WARN_ON(cpu_buffer, rb_list_head(list->prev->next) != list))
                return NULL;

        page = head = cpu_buffer->head_page;
        /*
         * It is possible that the writer moves the header behind
         * where we started, and we miss in one loop.
         * A second loop should grab the header, but we'll do
         * three loops just because I'm paranoid.
         */
        for (i = 0; i < 3; i++) {
                do {
                        if (rb_is_head_page(page, page->list.prev)) {
                                cpu_buffer->head_page = page;
                                return page;
                        }
                        rb_inc_page(&page);
                } while (page != head);
        }

        RB_WARN_ON(cpu_buffer, 1);

        return NULL;
}

static bool rb_head_page_replace(struct buffer_page *old,
                                struct buffer_page *new)
{
        unsigned long *ptr = (unsigned long *)&old->list.prev->next;
        unsigned long val;

        val = *ptr & ~RB_FLAG_MASK;
        val |= RB_PAGE_HEAD;

        return try_cmpxchg(ptr, &val, (unsigned long)&new->list);
}

/*
 * rb_tail_page_update - move the tail page forward
 */
static void rb_tail_page_update(struct ring_buffer_per_cpu *cpu_buffer,
                               struct buffer_page *tail_page,
                               struct buffer_page *next_page)
{
        unsigned long old_entries;
        unsigned long old_write;

        /*
         * The tail page now needs to be moved forward.
         *
         * We need to reset the tail page, but without messing
         * with possible erasing of data brought in by interrupts
         * that have moved the tail page and are currently on it.
         *
         * We add a counter to the write field to denote this.
         */
        old_write = local_add_return(RB_WRITE_INTCNT, &next_page->write);
        old_entries = local_add_return(RB_WRITE_INTCNT, &next_page->entries);

        /*
         * Just make sure we have seen our old_write and synchronize
         * with any interrupts that come in.
         */
        barrier();

        /*
         * If the tail page is still the same as what we think
         * it is, then it is up to us to update the tail
         * pointer.
         */
        if (tail_page == READ_ONCE(cpu_buffer->tail_page)) {
                /* Zero the write counter */
                unsigned long val = old_write & ~RB_WRITE_MASK;
                unsigned long eval = old_entries & ~RB_WRITE_MASK;

                /*
                 * This will only succeed if an interrupt did
                 * not come in and change it. In which case, we
                 * do not want to modify it.
                 *
                 * We add (void) to let the compiler know that we do not care
                 * about the return value of these functions. We use the
                 * cmpxchg to only update if an interrupt did not already
                 * do it for us. If the cmpxchg fails, we don't care.
                 */
                (void)local_cmpxchg(&next_page->write, old_write, val);
                (void)local_cmpxchg(&next_page->entries, old_entries, eval);

                /*
                 * No need to worry about races with clearing out the commit.
                 * it only can increment when a commit takes place. But that
                 * only happens in the outer most nested commit.
                 */
                local_set(&next_page->page->commit, 0);

                /* Either we update tail_page or an interrupt does */
                if (try_cmpxchg(&cpu_buffer->tail_page, &tail_page, next_page))
                        local_inc(&cpu_buffer->pages_touched);
        }
}

static void rb_check_bpage(struct ring_buffer_per_cpu *cpu_buffer,
                          struct buffer_page *bpage)
{
        unsigned long val = (unsigned long)bpage;

        RB_WARN_ON(cpu_buffer, val & RB_FLAG_MASK);
}

static bool rb_check_links(struct ring_buffer_per_cpu *cpu_buffer,
                           struct list_head *list)
{
        if (RB_WARN_ON(cpu_buffer,
                       rb_list_head(rb_list_head(list->next)->prev) != list))
                return false;

        if (RB_WARN_ON(cpu_buffer,
                       rb_list_head(rb_list_head(list->prev)->next) != list))
                return false;

        return true;
}

/**
 * rb_check_pages - integrity check of buffer pages
 * @cpu_buffer: CPU buffer with pages to test
 *
 * As a safety measure we check to make sure the data pages have not
 * been corrupted.
 */
static void rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
{
        struct list_head *head, *tmp;
        unsigned long buffer_cnt;
        unsigned long flags;
        int nr_loops = 0;

        /*
         * Walk the linked list underpinning the ring buffer and validate all
         * its next and prev links.
         *
         * The check acquires the reader_lock to avoid concurrent processing
         * with code that could be modifying the list. However, the lock cannot
         * be held for the entire duration of the walk, as this would make the
         * time when interrupts are disabled non-deterministic, dependent on the
         * ring buffer size. Therefore, the code releases and re-acquires the
         * lock after checking each page. The ring_buffer_per_cpu.cnt variable
         * is then used to detect if the list was modified while the lock was
         * not held, in which case the check needs to be restarted.
         *
         * The code attempts to perform the check at most three times before
         * giving up. This is acceptable because this is only a self-validation
         * to detect problems early on. In practice, the list modification
         * operations are fairly spaced, and so this check typically succeeds at
         * most on the second try.
         */
again:
        if (++nr_loops > 3)
                return;

        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
        head = rb_list_head(cpu_buffer->pages);
        if (!rb_check_links(cpu_buffer, head))
                goto out_locked;
        buffer_cnt = cpu_buffer->cnt;
        tmp = head;
        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);

        while (true) {
                raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);

                if (buffer_cnt != cpu_buffer->cnt) {
                        /* The list was updated, try again. */
                        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
                        goto again;
                }

                tmp = rb_list_head(tmp->next);
                if (tmp == head)
                        /* The iteration circled back, all is done. */
                        goto out_locked;

                if (!rb_check_links(cpu_buffer, tmp))
                        goto out_locked;

                raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
        }

out_locked:
        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
}

/*
 * Take an address, add the meta data size as well as the array of
 * array subbuffer indexes, then align it to a subbuffer size.
 *
 * This is used to help find the next per cpu subbuffer within a mapped range.
 */
static unsigned long
rb_range_align_subbuf(unsigned long addr, int subbuf_size, int nr_subbufs)
{
        addr += sizeof(struct ring_buffer_cpu_meta) +
                sizeof(int) * nr_subbufs;
        return ALIGN(addr, subbuf_size);
}

/*
 * Return the ring_buffer_meta for a given @cpu.
 */
static void *rb_range_meta(struct trace_buffer *buffer, int nr_pages, int cpu)
{
        int subbuf_size = buffer->subbuf_size + BUF_PAGE_HDR_SIZE;
        struct ring_buffer_cpu_meta *meta;
        struct ring_buffer_meta *bmeta;
        unsigned long ptr;
        int nr_subbufs;

        bmeta = buffer->meta;
        if (!bmeta)
                return NULL;

        ptr = (unsigned long)bmeta + bmeta->buffers_offset;
        meta = (struct ring_buffer_cpu_meta *)ptr;

        /* When nr_pages passed in is zero, the first meta has already been initialized */
        if (!nr_pages) {
                nr_subbufs = meta->nr_subbufs;
        } else {
                /* Include the reader page */
                nr_subbufs = nr_pages + 1;
        }

        /*
         * The first chunk may not be subbuffer aligned, where as
         * the rest of the chunks are.
         */
        if (cpu) {
                ptr = rb_range_align_subbuf(ptr, subbuf_size, nr_subbufs);
                ptr += subbuf_size * nr_subbufs;

                /* We can use multiplication to find chunks greater than 1 */
                if (cpu > 1) {
                        unsigned long size;
                        unsigned long p;

                        /* Save the beginning of this CPU chunk */
                        p = ptr;
                        ptr = rb_range_align_subbuf(ptr, subbuf_size, nr_subbufs);
                        ptr += subbuf_size * nr_subbufs;

                        /* Now all chunks after this are the same size */
                        size = ptr - p;
                        ptr += size * (cpu - 2);
                }
        }
        return (void *)ptr;
}

/* Return the start of subbufs given the meta pointer */
static void *rb_subbufs_from_meta(struct ring_buffer_cpu_meta *meta)
{
        int subbuf_size = meta->subbuf_size;
        unsigned long ptr;

        ptr = (unsigned long)meta;
        ptr = rb_range_align_subbuf(ptr, subbuf_size, meta->nr_subbufs);

        return (void *)ptr;
}

/*
 * Return a specific sub-buffer for a given @cpu defined by @idx.
 */
static void *rb_range_buffer(struct ring_buffer_per_cpu *cpu_buffer, int idx)
{
        struct ring_buffer_cpu_meta *meta;
        unsigned long ptr;
        int subbuf_size;

        meta = rb_range_meta(cpu_buffer->buffer, 0, cpu_buffer->cpu);
        if (!meta)
                return NULL;

        if (WARN_ON_ONCE(idx >= meta->nr_subbufs))
                return NULL;

        subbuf_size = meta->subbuf_size;

        /* Map this buffer to the order that's in meta->buffers[] */
        idx = meta->buffers[idx];

        ptr = (unsigned long)rb_subbufs_from_meta(meta);

        ptr += subbuf_size * idx;
        if (ptr + subbuf_size > cpu_buffer->buffer->range_addr_end)
                return NULL;

        return (void *)ptr;
}

/*
 * See if the existing memory contains a valid meta section.
 * if so, use that, otherwise initialize it.
 */
static bool rb_meta_init(struct trace_buffer *buffer, int scratch_size)
{
        unsigned long ptr = buffer->range_addr_start;
        struct ring_buffer_meta *bmeta;
        unsigned long total_size;
        int struct_sizes;

        bmeta = (struct ring_buffer_meta *)ptr;
        buffer->meta = bmeta;

        total_size = buffer->range_addr_end - buffer->range_addr_start;

        struct_sizes = sizeof(struct ring_buffer_cpu_meta);
        struct_sizes |= sizeof(*bmeta) << 16;

        /* The first buffer will start word size after the meta page */
        ptr += sizeof(*bmeta);
        ptr = ALIGN(ptr, sizeof(long));
        ptr += scratch_size;

        if (bmeta->magic != RING_BUFFER_META_MAGIC) {
                pr_info("Ring buffer boot meta mismatch of magic\n");
                goto init;
        }

        if (bmeta->struct_sizes != struct_sizes) {
                pr_info("Ring buffer boot meta mismatch of struct size\n");
                goto init;
        }

        if (bmeta->total_size != total_size) {
                pr_info("Ring buffer boot meta mismatch of total size\n");
                goto init;
        }

        if (bmeta->buffers_offset > bmeta->total_size) {
                pr_info("Ring buffer boot meta mismatch of offset outside of total size\n");
                goto init;
        }

        if (bmeta->buffers_offset != (void *)ptr - (void *)bmeta) {
                pr_info("Ring buffer boot meta mismatch of first buffer offset\n");
                goto init;
        }

        return true;

 init:
        bmeta->magic = RING_BUFFER_META_MAGIC;
        bmeta->struct_sizes = struct_sizes;
        bmeta->total_size = total_size;
        bmeta->buffers_offset = (void *)ptr - (void *)bmeta;

        /* Zero out the scatch pad */
        memset((void *)bmeta + sizeof(*bmeta), 0, bmeta->buffers_offset - sizeof(*bmeta));

        return false;
}

/*
 * See if the existing memory contains valid ring buffer data.
 * As the previous kernel must be the same as this kernel, all
 * the calculations (size of buffers and number of buffers)
 * must be the same.
 */
static bool rb_cpu_meta_valid(struct ring_buffer_cpu_meta *meta, int cpu,
                              struct trace_buffer *buffer, int nr_pages,
                              unsigned long *subbuf_mask)
{
        int subbuf_size = PAGE_SIZE;
        struct buffer_data_page *subbuf;
        unsigned long buffers_start;
        unsigned long buffers_end;
        int i;

        if (!subbuf_mask)
                return false;

        buffers_start = meta->first_buffer;
        buffers_end = meta->first_buffer + (subbuf_size * meta->nr_subbufs);

        /* Is the head and commit buffers within the range of buffers? */
        if (meta->head_buffer < buffers_start ||
            meta->head_buffer >= buffers_end) {
                pr_info("Ring buffer boot meta [%d] head buffer out of range\n", cpu);
                return false;
        }

        if (meta->commit_buffer < buffers_start ||
            meta->commit_buffer >= buffers_end) {
                pr_info("Ring buffer boot meta [%d] commit buffer out of range\n", cpu);
                return false;
        }

        subbuf = rb_subbufs_from_meta(meta);

        bitmap_clear(subbuf_mask, 0, meta->nr_subbufs);

        /* Is the meta buffers and the subbufs themselves have correct data? */
        for (i = 0; i < meta->nr_subbufs; i++) {
                if (meta->buffers[i] < 0 ||
                    meta->buffers[i] >= meta->nr_subbufs) {
                        pr_info("Ring buffer boot meta [%d] array out of range\n", cpu);
                        return false;
                }

                if ((unsigned)local_read(&subbuf->commit) > subbuf_size) {
                        pr_info("Ring buffer boot meta [%d] buffer invalid commit\n", cpu);
                        return false;
                }

                if (test_bit(meta->buffers[i], subbuf_mask)) {
                        pr_info("Ring buffer boot meta [%d] array has duplicates\n", cpu);
                        return false;
                }

                set_bit(meta->buffers[i], subbuf_mask);
                subbuf = (void *)subbuf + subbuf_size;
        }

        return true;
}

static int rb_meta_subbuf_idx(struct ring_buffer_cpu_meta *meta, void *subbuf);

static int rb_read_data_buffer(struct buffer_data_page *dpage, int tail, int cpu,
                               unsigned long long *timestamp, u64 *delta_ptr)
{
        struct ring_buffer_event *event;
        u64 ts, delta;
        int events = 0;
        int e;

        *delta_ptr = 0;
        *timestamp = 0;

        ts = dpage->time_stamp;

        for (e = 0; e < tail; e += rb_event_length(event)) {

                event = (struct ring_buffer_event *)(dpage->data + e);

                switch (event->type_len) {

                case RINGBUF_TYPE_TIME_EXTEND:
                        delta = rb_event_time_stamp(event);
                        ts += delta;
                        break;

                case RINGBUF_TYPE_TIME_STAMP:
                        delta = rb_event_time_stamp(event);
                        delta = rb_fix_abs_ts(delta, ts);
                        if (delta < ts) {
                                *delta_ptr = delta;
                                *timestamp = ts;
                                return -1;
                        }
                        ts = delta;
                        break;

                case RINGBUF_TYPE_PADDING:
                        if (event->time_delta == 1)
                                break;
                        fallthrough;
                case RINGBUF_TYPE_DATA:
                        events++;
                        ts += event->time_delta;
                        break;

                default:
                        return -1;
                }
        }
        *timestamp = ts;
        return events;
}

static int rb_validate_buffer(struct buffer_data_page *dpage, int cpu)
{
        unsigned long long ts;
        u64 delta;
        int tail;

        tail = local_read(&dpage->commit);
        return rb_read_data_buffer(dpage, tail, cpu, &ts, &delta);
}

/* If the meta data has been validated, now validate the events */
static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer)
{
        struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta;
        struct buffer_page *head_page;
        unsigned long entry_bytes = 0;
        unsigned long entries = 0;
        int ret;
        int i;

        if (!meta || !meta->head_buffer)
                return;

        /* Do the reader page first */
        ret = rb_validate_buffer(cpu_buffer->reader_page->page, cpu_buffer->cpu);
        if (ret < 0) {
                pr_info("Ring buffer reader page is invalid\n");
                goto invalid;
        }
        entries += ret;
        entry_bytes += local_read(&cpu_buffer->reader_page->page->commit);
        local_set(&cpu_buffer->reader_page->entries, ret);

        head_page = cpu_buffer->head_page;

        /* If both the head and commit are on the reader_page then we are done. */
        if (head_page == cpu_buffer->reader_page &&
            head_page == cpu_buffer->commit_page)
                goto done;

        /* Iterate until finding the commit page */
        for (i = 0; i < meta->nr_subbufs + 1; i++, rb_inc_page(&head_page)) {

                /* Reader page has already been done */
                if (head_page == cpu_buffer->reader_page)
                        continue;

                ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu);
                if (ret < 0) {
                        pr_info("Ring buffer meta [%d] invalid buffer page\n",
                                cpu_buffer->cpu);
                        goto invalid;
                }

                /* If the buffer has content, update pages_touched */
                if (ret)
                        local_inc(&cpu_buffer->pages_touched);

                entries += ret;
                entry_bytes += local_read(&head_page->page->commit);
                local_set(&cpu_buffer->head_page->entries, ret);

                if (head_page == cpu_buffer->commit_page)
                        break;
        }

        if (head_page != cpu_buffer->commit_page) {
                pr_info("Ring buffer meta [%d] commit page not found\n",
                        cpu_buffer->cpu);
                goto invalid;
        }
 done:
        local_set(&cpu_buffer->entries, entries);
        local_set(&cpu_buffer->entries_bytes, entry_bytes);

        pr_info("Ring buffer meta [%d] is from previous boot!\n", cpu_buffer->cpu);
        return;

 invalid:
        /* The content of the buffers are invalid, reset the meta data */
        meta->head_buffer = 0;
        meta->commit_buffer = 0;

        /* Reset the reader page */
        local_set(&cpu_buffer->reader_page->entries, 0);
        local_set(&cpu_buffer->reader_page->page->commit, 0);

        /* Reset all the subbuffers */
        for (i = 0; i < meta->nr_subbufs - 1; i++, rb_inc_page(&head_page)) {
                local_set(&head_page->entries, 0);
                local_set(&head_page->page->commit, 0);
        }
}

static void rb_range_meta_init(struct trace_buffer *buffer, int nr_pages, int scratch_size)
{
        struct ring_buffer_cpu_meta *meta;
        unsigned long *subbuf_mask;
        unsigned long delta;
        void *subbuf;
        bool valid = false;
        int cpu;
        int i;

        /* Create a mask to test the subbuf array */
        subbuf_mask = bitmap_alloc(nr_pages + 1, GFP_KERNEL);
        /* If subbuf_mask fails to allocate, then rb_meta_valid() will return false */

        if (rb_meta_init(buffer, scratch_size))
                valid = true;

        for (cpu = 0; cpu < nr_cpu_ids; cpu++) {
                void *next_meta;

                meta = rb_range_meta(buffer, nr_pages, cpu);

                if (valid && rb_cpu_meta_valid(meta, cpu, buffer, nr_pages, subbuf_mask)) {
                        /* Make the mappings match the current address */
                        subbuf = rb_subbufs_from_meta(meta);
                        delta = (unsigned long)subbuf - meta->first_buffer;
                        meta->first_buffer += delta;
                        meta->head_buffer += delta;
                        meta->commit_buffer += delta;
                        continue;
                }

                if (cpu < nr_cpu_ids - 1)
                        next_meta = rb_range_meta(buffer, nr_pages, cpu + 1);
                else
                        next_meta = (void *)buffer->range_addr_end;

                memset(meta, 0, next_meta - (void *)meta);

                meta->nr_subbufs = nr_pages + 1;
                meta->subbuf_size = PAGE_SIZE;

                subbuf = rb_subbufs_from_meta(meta);

                meta->first_buffer = (unsigned long)subbuf;

                /*
                 * The buffers[] array holds the order of the sub-buffers
                 * that are after the meta data. The sub-buffers may
                 * be swapped out when read and inserted into a different
                 * location of the ring buffer. Although their addresses
                 * remain the same, the buffers[] array contains the
                 * index into the sub-buffers holding their actual order.
                 */
                for (i = 0; i < meta->nr_subbufs; i++) {
                        meta->buffers[i] = i;
                        rb_init_page(subbuf);
                        subbuf += meta->subbuf_size;
                }
        }
        bitmap_free(subbuf_mask);
}

static void *rbm_start(struct seq_file *m, loff_t *pos)
{
        struct ring_buffer_per_cpu *cpu_buffer = m->private;
        struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta;
        unsigned long val;

        if (!meta)
                return NULL;

        if (*pos > meta->nr_subbufs)
                return NULL;

        val = *pos;
        val++;

        return (void *)val;
}

static void *rbm_next(struct seq_file *m, void *v, loff_t *pos)
{
        (*pos)++;

        return rbm_start(m, pos);
}

static int rbm_show(struct seq_file *m, void *v)
{
        struct ring_buffer_per_cpu *cpu_buffer = m->private;
        struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta;
        unsigned long val = (unsigned long)v;

        if (val == 1) {
                seq_printf(m, "head_buffer:   %d\n",
                           rb_meta_subbuf_idx(meta, (void *)meta->head_buffer));
                seq_printf(m, "commit_buffer: %d\n",
                           rb_meta_subbuf_idx(meta, (void *)meta->commit_buffer));
                seq_printf(m, "subbuf_size:   %d\n", meta->subbuf_size);
                seq_printf(m, "nr_subbufs:    %d\n", meta->nr_subbufs);
                return 0;
        }

        val -= 2;
        seq_printf(m, "buffer[%ld]:    %d\n", val, meta->buffers[val]);

        return 0;
}

static void rbm_stop(struct seq_file *m, void *p)
{
}

static const struct seq_operations rb_meta_seq_ops = {
        .start                = rbm_start,
        .next                = rbm_next,
        .show                = rbm_show,
        .stop                = rbm_stop,
};

int ring_buffer_meta_seq_init(struct file *file, struct trace_buffer *buffer, int cpu)
{
        struct seq_file *m;
        int ret;

        ret = seq_open(file, &rb_meta_seq_ops);
        if (ret)
                return ret;

        m = file->private_data;
        m->private = buffer->buffers[cpu];

        return 0;
}

/* Map the buffer_pages to the previous head and commit pages */
static void rb_meta_buffer_update(struct ring_buffer_per_cpu *cpu_buffer,
                                  struct buffer_page *bpage)
{
        struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta;

        if (meta->head_buffer == (unsigned long)bpage->page)
                cpu_buffer->head_page = bpage;

        if (meta->commit_buffer == (unsigned long)bpage->page) {
                cpu_buffer->commit_page = bpage;
                cpu_buffer->tail_page = bpage;
        }
}

static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
                long nr_pages, struct list_head *pages)
{
        struct trace_buffer *buffer = cpu_buffer->buffer;
        struct ring_buffer_cpu_meta *meta = NULL;
        struct buffer_page *bpage, *tmp;
        bool user_thread = current->mm != NULL;
        gfp_t mflags;
        long i;

        /*
         * Check if the available memory is there first.
         * Note, si_mem_available() only gives us a rough estimate of available
         * memory. It may not be accurate. But we don't care, we just want
         * to prevent doing any allocation when it is obvious that it is
         * not going to succeed.
         */
        i = si_mem_available();
        if (i < nr_pages)
                return -ENOMEM;

        /*
         * __GFP_RETRY_MAYFAIL flag makes sure that the allocation fails
         * gracefully without invoking oom-killer and the system is not
         * destabilized.
         */
        mflags = GFP_KERNEL | __GFP_RETRY_MAYFAIL;

        /*
         * If a user thread allocates too much, and si_mem_available()
         * reports there's enough memory, even though there is not.
         * Make sure the OOM killer kills this thread. This can happen
         * even with RETRY_MAYFAIL because another task may be doing
         * an allocation after this task has taken all memory.
         * This is the task the OOM killer needs to take out during this
         * loop, even if it was triggered by an allocation somewhere else.
         */
        if (user_thread)
                set_current_oom_origin();

        if (buffer->range_addr_start)
                meta = rb_range_meta(buffer, nr_pages, cpu_buffer->cpu);

        for (i = 0; i < nr_pages; i++) {
                struct page *page;

                bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
                                    mflags, cpu_to_node(cpu_buffer->cpu));
                if (!bpage)
                        goto free_pages;

                rb_check_bpage(cpu_buffer, bpage);

                /*
                 * Append the pages as for mapped buffers we want to keep
                 * the order
                 */
                list_add_tail(&bpage->list, pages);

                if (meta) {
                        /* A range was given. Use that for the buffer page */
                        bpage->page = rb_range_buffer(cpu_buffer, i + 1);
                        if (!bpage->page)
                                goto free_pages;
                        /* If this is valid from a previous boot */
                        if (meta->head_buffer)
                                rb_meta_buffer_update(cpu_buffer, bpage);
                        bpage->range = 1;
                        bpage->id = i + 1;
                } else {
                        page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu),
                                                mflags | __GFP_COMP | __GFP_ZERO,
                                                cpu_buffer->buffer->subbuf_order);
                        if (!page)
                                goto free_pages;
                        bpage->page = page_address(page);
                        rb_init_page(bpage->page);
                }
                bpage->order = cpu_buffer->buffer->subbuf_order;

                if (user_thread && fatal_signal_pending(current))
                        goto free_pages;
        }
        if (user_thread)
                clear_current_oom_origin();

        return 0;

free_pages:
        list_for_each_entry_safe(bpage, tmp, pages, list) {
                list_del_init(&bpage->list);
                free_buffer_page(bpage);
        }
        if (user_thread)
                clear_current_oom_origin();

        return -ENOMEM;
}

static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
                             unsigned long nr_pages)
{
        LIST_HEAD(pages);

        WARN_ON(!nr_pages);

        if (__rb_allocate_pages(cpu_buffer, nr_pages, &pages))
                return -ENOMEM;

        /*
         * The ring buffer page list is a circular list that does not
         * start and end with a list head. All page list items point to
         * other pages.
         */
        cpu_buffer->pages = pages.next;
        list_del(&pages);

        cpu_buffer->nr_pages = nr_pages;

        rb_check_pages(cpu_buffer);

        return 0;
}

static struct ring_buffer_per_cpu *
rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        struct ring_buffer_cpu_meta *meta;
        struct buffer_page *bpage;
        struct page *page;
        int ret;

        cpu_buffer = kzalloc_node(ALIGN(sizeof(*cpu_buffer), cache_line_size()),
                                  GFP_KERNEL, cpu_to_node(cpu));
        if (!cpu_buffer)
                return NULL;

        cpu_buffer->cpu = cpu;
        cpu_buffer->buffer = buffer;
        raw_spin_lock_init(&cpu_buffer->reader_lock);
        lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key);
        cpu_buffer->lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
        INIT_WORK(&cpu_buffer->update_pages_work, update_pages_handler);
        init_completion(&cpu_buffer->update_done);
        init_irq_work(&cpu_buffer->irq_work.work, rb_wake_up_waiters);
        init_waitqueue_head(&cpu_buffer->irq_work.waiters);
        init_waitqueue_head(&cpu_buffer->irq_work.full_waiters);
        mutex_init(&cpu_buffer->mapping_lock);

        bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
                            GFP_KERNEL, cpu_to_node(cpu));
        if (!bpage)
                goto fail_free_buffer;

        rb_check_bpage(cpu_buffer, bpage);

        cpu_buffer->reader_page = bpage;

        if (buffer->range_addr_start) {
                /*
                 * Range mapped buffers have the same restrictions as memory
                 * mapped ones do.
                 */
                cpu_buffer->mapped = 1;
                cpu_buffer->ring_meta = rb_range_meta(buffer, nr_pages, cpu);
                bpage->page = rb_range_buffer(cpu_buffer, 0);
                if (!bpage->page)
                        goto fail_free_reader;
                if (cpu_buffer->ring_meta->head_buffer)
                        rb_meta_buffer_update(cpu_buffer, bpage);
                bpage->range = 1;
        } else {
                page = alloc_pages_node(cpu_to_node(cpu),
                                        GFP_KERNEL | __GFP_COMP | __GFP_ZERO,
                                        cpu_buffer->buffer->subbuf_order);
                if (!page)
                        goto fail_free_reader;
                bpage->page = page_address(page);
                rb_init_page(bpage->page);
        }

        INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
        INIT_LIST_HEAD(&cpu_buffer->new_pages);

        ret = rb_allocate_pages(cpu_buffer, nr_pages);
        if (ret < 0)
                goto fail_free_reader;

        rb_meta_validate_events(cpu_buffer);

        /* If the boot meta was valid then this has already been updated */
        meta = cpu_buffer->ring_meta;
        if (!meta || !meta->head_buffer ||
            !cpu_buffer->head_page || !cpu_buffer->commit_page || !cpu_buffer->tail_page) {
                if (meta && meta->head_buffer &&
                    (cpu_buffer->head_page || cpu_buffer->commit_page || cpu_buffer->tail_page)) {
                        pr_warn("Ring buffer meta buffers not all mapped\n");
                        if (!cpu_buffer->head_page)
                                pr_warn("   Missing head_page\n");
                        if (!cpu_buffer->commit_page)
                                pr_warn("   Missing commit_page\n");
                        if (!cpu_buffer->tail_page)
                                pr_warn("   Missing tail_page\n");
                }

                cpu_buffer->head_page
                        = list_entry(cpu_buffer->pages, struct buffer_page, list);
                cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page;

                rb_head_page_activate(cpu_buffer);

                if (cpu_buffer->ring_meta)
                        meta->commit_buffer = meta->head_buffer;
        } else {
                /* The valid meta buffer still needs to activate the head page */
                rb_head_page_activate(cpu_buffer);
        }

        return cpu_buffer;

 fail_free_reader:
        free_buffer_page(cpu_buffer->reader_page);

 fail_free_buffer:
        kfree(cpu_buffer);
        return NULL;
}

static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer)
{
        struct list_head *head = cpu_buffer->pages;
        struct buffer_page *bpage, *tmp;

        irq_work_sync(&cpu_buffer->irq_work.work);

        free_buffer_page(cpu_buffer->reader_page);

        if (head) {
                rb_head_page_deactivate(cpu_buffer);

                list_for_each_entry_safe(bpage, tmp, head, list) {
                        list_del_init(&bpage->list);
                        free_buffer_page(bpage);
                }
                bpage = list_entry(head, struct buffer_page, list);
                free_buffer_page(bpage);
        }

        free_page((unsigned long)cpu_buffer->free_page);

        kfree(cpu_buffer);
}

static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags,
                                         int order, unsigned long start,
                                         unsigned long end,
                                         unsigned long scratch_size,
                                         struct lock_class_key *key)
{
        struct trace_buffer *buffer;
        long nr_pages;
        int subbuf_size;
        int bsize;
        int cpu;
        int ret;

        /* keep it in its own cache line */
        buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()),
                         GFP_KERNEL);
        if (!buffer)
                return NULL;

        if (!zalloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))
                goto fail_free_buffer;

        buffer->subbuf_order = order;
        subbuf_size = (PAGE_SIZE << order);
        buffer->subbuf_size = subbuf_size - BUF_PAGE_HDR_SIZE;

        /* Max payload is buffer page size - header (8bytes) */
        buffer->max_data_size = buffer->subbuf_size - (sizeof(u32) * 2);

        buffer->flags = flags;
        buffer->clock = trace_clock_local;
        buffer->reader_lock_key = key;

        init_irq_work(&buffer->irq_work.work, rb_wake_up_waiters);
        init_waitqueue_head(&buffer->irq_work.waiters);

        buffer->cpus = nr_cpu_ids;

        bsize = sizeof(void *) * nr_cpu_ids;
        buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()),
                                  GFP_KERNEL);
        if (!buffer->buffers)
                goto fail_free_cpumask;

        /* If start/end are specified, then that overrides size */
        if (start && end) {
                unsigned long buffers_start;
                unsigned long ptr;
                int n;

                /* Make sure that start is word aligned */
                start = ALIGN(start, sizeof(long));

                /* scratch_size needs to be aligned too */
                scratch_size = ALIGN(scratch_size, sizeof(long));

                /* Subtract the buffer meta data and word aligned */
                buffers_start = start + sizeof(struct ring_buffer_cpu_meta);
                buffers_start = ALIGN(buffers_start, sizeof(long));
                buffers_start += scratch_size;

                /* Calculate the size for the per CPU data */
                size = end - buffers_start;
                size = size / nr_cpu_ids;

                /*
                 * The number of sub-buffers (nr_pages) is determined by the
                 * total size allocated minus the meta data size.
                 * Then that is divided by the number of per CPU buffers
                 * needed, plus account for the integer array index that
                 * will be appended to the meta data.
                 */
                nr_pages = (size - sizeof(struct ring_buffer_cpu_meta)) /
                        (subbuf_size + sizeof(int));
                /* Need at least two pages plus the reader page */
                if (nr_pages < 3)
                        goto fail_free_buffers;

 again:
                /* Make sure that the size fits aligned */
                for (n = 0, ptr = buffers_start; n < nr_cpu_ids; n++) {
                        ptr += sizeof(struct ring_buffer_cpu_meta) +
                                sizeof(int) * nr_pages;
                        ptr = ALIGN(ptr, subbuf_size);
                        ptr += subbuf_size * nr_pages;
                }
                if (ptr > end) {
                        if (nr_pages <= 3)
                                goto fail_free_buffers;
                        nr_pages--;
                        goto again;
                }

                /* nr_pages should not count the reader page */
                nr_pages--;
                buffer->range_addr_start = start;
                buffer->range_addr_end = end;

                rb_range_meta_init(buffer, nr_pages, scratch_size);
        } else {

                /* need at least two pages */
                nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size);
                if (nr_pages < 2)
                        nr_pages = 2;
        }

        cpu = raw_smp_processor_id();
        cpumask_set_cpu(cpu, buffer->cpumask);
        buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, nr_pages, cpu);
        if (!buffer->buffers[cpu])
                goto fail_free_buffers;

        ret = cpuhp_state_add_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node);
        if (ret < 0)
                goto fail_free_buffers;

        mutex_init(&buffer->mutex);

        return buffer;

 fail_free_buffers:
        for_each_buffer_cpu(buffer, cpu) {
                if (buffer->buffers[cpu])
                        rb_free_cpu_buffer(buffer->buffers[cpu]);
        }
        kfree(buffer->buffers);

 fail_free_cpumask:
        free_cpumask_var(buffer->cpumask);

 fail_free_buffer:
        kfree(buffer);
        return NULL;
}

/**
 * __ring_buffer_alloc - allocate a new ring_buffer
 * @size: the size in bytes per cpu that is needed.
 * @flags: attributes to set for the ring buffer.
 * @key: ring buffer reader_lock_key.
 *
 * Currently the only flag that is available is the RB_FL_OVERWRITE
 * flag. This flag means that the buffer will overwrite old data
 * when the buffer wraps. If this flag is not set, the buffer will
 * drop data when the tail hits the head.
 */
struct trace_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
                                        struct lock_class_key *key)
{
        /* Default buffer page size - one system page */
        return alloc_buffer(size, flags, 0, 0, 0, 0, key);

}
EXPORT_SYMBOL_GPL(__ring_buffer_alloc);

/**
 * __ring_buffer_alloc_range - allocate a new ring_buffer from existing memory
 * @size: the size in bytes per cpu that is needed.
 * @flags: attributes to set for the ring buffer.
 * @order: sub-buffer order
 * @start: start of allocated range
 * @range_size: size of allocated range
 * @scratch_size: size of scratch area (for preallocated memory buffers)
 * @key: ring buffer reader_lock_key.
 *
 * Currently the only flag that is available is the RB_FL_OVERWRITE
 * flag. This flag means that the buffer will overwrite old data
 * when the buffer wraps. If this flag is not set, the buffer will
 * drop data when the tail hits the head.
 */
struct trace_buffer *__ring_buffer_alloc_range(unsigned long size, unsigned flags,
                                               int order, unsigned long start,
                                               unsigned long range_size,
                                               unsigned long scratch_size,
                                               struct lock_class_key *key)
{
        return alloc_buffer(size, flags, order, start, start + range_size,
                            scratch_size, key);
}

void *ring_buffer_meta_scratch(struct trace_buffer *buffer, unsigned int *size)
{
        struct ring_buffer_meta *meta;
        void *ptr;

        if (!buffer || !buffer->meta)
                return NULL;

        meta = buffer->meta;

        ptr = (void *)ALIGN((unsigned long)meta + sizeof(*meta), sizeof(long));

        if (size)
                *size = (void *)meta + meta->buffers_offset - ptr;

        return ptr;
}

/**
 * ring_buffer_free - free a ring buffer.
 * @buffer: the buffer to free.
 */
void
ring_buffer_free(struct trace_buffer *buffer)
{
        int cpu;

        cpuhp_state_remove_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node);

        irq_work_sync(&buffer->irq_work.work);

        for_each_buffer_cpu(buffer, cpu)
                rb_free_cpu_buffer(buffer->buffers[cpu]);

        kfree(buffer->buffers);
        free_cpumask_var(buffer->cpumask);

        kfree(buffer);
}
EXPORT_SYMBOL_GPL(ring_buffer_free);

void ring_buffer_set_clock(struct trace_buffer *buffer,
                           u64 (*clock)(void))
{
        buffer->clock = clock;
}

void ring_buffer_set_time_stamp_abs(struct trace_buffer *buffer, bool abs)
{
        buffer->time_stamp_abs = abs;
}

bool ring_buffer_time_stamp_abs(struct trace_buffer *buffer)
{
        return buffer->time_stamp_abs;
}

static inline unsigned long rb_page_entries(struct buffer_page *bpage)
{
        return local_read(&bpage->entries) & RB_WRITE_MASK;
}

static inline unsigned long rb_page_write(struct buffer_page *bpage)
{
        return local_read(&bpage->write) & RB_WRITE_MASK;
}

static bool
rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned long nr_pages)
{
        struct list_head *tail_page, *to_remove, *next_page;
        struct buffer_page *to_remove_page, *tmp_iter_page;
        struct buffer_page *last_page, *first_page;
        unsigned long nr_removed;
        unsigned long head_bit;
        int page_entries;

        head_bit = 0;

        raw_spin_lock_irq(&cpu_buffer->reader_lock);
        atomic_inc(&cpu_buffer->record_disabled);
        /*
         * We don't race with the readers since we have acquired the reader
         * lock. We also don't race with writers after disabling recording.
         * This makes it easy to figure out the first and the last page to be
         * removed from the list. We unlink all the pages in between including
         * the first and last pages. This is done in a busy loop so that we
         * lose the least number of traces.
         * The pages are freed after we restart recording and unlock readers.
         */
        tail_page = &cpu_buffer->tail_page->list;

        /*
         * tail page might be on reader page, we remove the next page
         * from the ring buffer
         */
        if (cpu_buffer->tail_page == cpu_buffer->reader_page)
                tail_page = rb_list_head(tail_page->next);
        to_remove = tail_page;

        /* start of pages to remove */
        first_page = list_entry(rb_list_head(to_remove->next),
                                struct buffer_page, list);

        for (nr_removed = 0; nr_removed < nr_pages; nr_removed++) {
                to_remove = rb_list_head(to_remove)->next;
                head_bit |= (unsigned long)to_remove & RB_PAGE_HEAD;
        }
        /* Read iterators need to reset themselves when some pages removed */
        cpu_buffer->pages_removed += nr_removed;

        next_page = rb_list_head(to_remove)->next;

        /*
         * Now we remove all pages between tail_page and next_page.
         * Make sure that we have head_bit value preserved for the
         * next page
         */
        tail_page->next = (struct list_head *)((unsigned long)next_page |
                                                head_bit);
        next_page = rb_list_head(next_page);
        next_page->prev = tail_page;

        /* make sure pages points to a valid page in the ring buffer */
        cpu_buffer->pages = next_page;
        cpu_buffer->cnt++;

        /* update head page */
        if (head_bit)
                cpu_buffer->head_page = list_entry(next_page,
                                                struct buffer_page, list);

        /* pages are removed, resume tracing and then free the pages */
        atomic_dec(&cpu_buffer->record_disabled);
        raw_spin_unlock_irq(&cpu_buffer->reader_lock);

        RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages));

        /* last buffer page to remove */
        last_page = list_entry(rb_list_head(to_remove), struct buffer_page,
                                list);
        tmp_iter_page = first_page;

        do {
                cond_resched();

                to_remove_page = tmp_iter_page;
                rb_inc_page(&tmp_iter_page);

                /* update the counters */
                page_entries = rb_page_entries(to_remove_page);
                if (page_entries) {
                        /*
                         * If something was added to this page, it was full
                         * since it is not the tail page. So we deduct the
                         * bytes consumed in ring buffer from here.
                         * Increment overrun to account for the lost events.
                         */
                        local_add(page_entries, &cpu_buffer->overrun);
                        local_sub(rb_page_commit(to_remove_page), &cpu_buffer->entries_bytes);
                        local_inc(&cpu_buffer->pages_lost);
                }

                /*
                 * We have already removed references to this list item, just
                 * free up the buffer_page and its page
                 */
                free_buffer_page(to_remove_page);
                nr_removed--;

        } while (to_remove_page != last_page);

        RB_WARN_ON(cpu_buffer, nr_removed);

        return nr_removed == 0;
}

static bool
rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer)
{
        struct list_head *pages = &cpu_buffer->new_pages;
        unsigned long flags;
        bool success;
        int retries;

        /* Can be called at early boot up, where interrupts must not been enabled */
        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
        /*
         * We are holding the reader lock, so the reader page won't be swapped
         * in the ring buffer. Now we are racing with the writer trying to
         * move head page and the tail page.
         * We are going to adapt the reader page update process where:
         * 1. We first splice the start and end of list of new pages between
         *    the head page and its previous page.
         * 2. We cmpxchg the prev_page->next to point from head page to the
         *    start of new pages list.
         * 3. Finally, we update the head->prev to the end of new list.
         *
         * We will try this process 10 times, to make sure that we don't keep
         * spinning.
         */
        retries = 10;
        success = false;
        while (retries--) {
                struct list_head *head_page, *prev_page;
                struct list_head *last_page, *first_page;
                struct list_head *head_page_with_bit;
                struct buffer_page *hpage = rb_set_head_page(cpu_buffer);

                if (!hpage)
                        break;
                head_page = &hpage->list;
                prev_page = head_page->prev;

                first_page = pages->next;
                last_page  = pages->prev;

                head_page_with_bit = (struct list_head *)
                                     ((unsigned long)head_page | RB_PAGE_HEAD);

                last_page->next = head_page_with_bit;
                first_page->prev = prev_page;

                /* caution: head_page_with_bit gets updated on cmpxchg failure */
                if (try_cmpxchg(&prev_page->next,
                                &head_page_with_bit, first_page)) {
                        /*
                         * yay, we replaced the page pointer to our new list,
                         * now, we just have to update to head page's prev
                         * pointer to point to end of list
                         */
                        head_page->prev = last_page;
                        cpu_buffer->cnt++;
                        success = true;
                        break;
                }
        }

        if (success)
                INIT_LIST_HEAD(pages);
        /*
         * If we weren't successful in adding in new pages, warn and stop
         * tracing
         */
        RB_WARN_ON(cpu_buffer, !success);
        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);

        /* free pages if they weren't inserted */
        if (!success) {
                struct buffer_page *bpage, *tmp;
                list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,
                                         list) {
                        list_del_init(&bpage->list);
                        free_buffer_page(bpage);
                }
        }
        return success;
}

static void rb_update_pages(struct ring_buffer_per_cpu *cpu_buffer)
{
        bool success;

        if (cpu_buffer->nr_pages_to_update > 0)
                success = rb_insert_pages(cpu_buffer);
        else
                success = rb_remove_pages(cpu_buffer,
                                        -cpu_buffer->nr_pages_to_update);

        if (success)
                cpu_buffer->nr_pages += cpu_buffer->nr_pages_to_update;
}

static void update_pages_handler(struct work_struct *work)
{
        struct ring_buffer_per_cpu *cpu_buffer = container_of(work,
                        struct ring_buffer_per_cpu, update_pages_work);
        rb_update_pages(cpu_buffer);
        complete(&cpu_buffer->update_done);
}

/**
 * ring_buffer_resize - resize the ring buffer
 * @buffer: the buffer to resize.
 * @size: the new size.
 * @cpu_id: the cpu buffer to resize
 *
 * Minimum size is 2 * buffer->subbuf_size.
 *
 * Returns 0 on success and < 0 on failure.
 */
int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size,
                        int cpu_id)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        unsigned long nr_pages;
        int cpu, err;

        /*
         * Always succeed at resizing a non-existent buffer:
         */
        if (!buffer)
                return 0;

        /* Make sure the requested buffer exists */
        if (cpu_id != RING_BUFFER_ALL_CPUS &&
            !cpumask_test_cpu(cpu_id, buffer->cpumask))
                return 0;

        nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size);

        /* we need a minimum of two pages */
        if (nr_pages < 2)
                nr_pages = 2;

        /* prevent another thread from changing buffer sizes */
        mutex_lock(&buffer->mutex);
        atomic_inc(&buffer->resizing);

        if (cpu_id == RING_BUFFER_ALL_CPUS) {
                /*
                 * Don't succeed if resizing is disabled, as a reader might be
                 * manipulating the ring buffer and is expecting a sane state while
                 * this is true.
                 */
                for_each_buffer_cpu(buffer, cpu) {
                        cpu_buffer = buffer->buffers[cpu];
                        if (atomic_read(&cpu_buffer->resize_disabled)) {
                                err = -EBUSY;
                                goto out_err_unlock;
                        }
                }

                /* calculate the pages to update */
                for_each_buffer_cpu(buffer, cpu) {
                        cpu_buffer = buffer->buffers[cpu];

                        cpu_buffer->nr_pages_to_update = nr_pages -
                                                        cpu_buffer->nr_pages;
                        /*
                         * nothing more to do for removing pages or no update
                         */
                        if (cpu_buffer->nr_pages_to_update <= 0)
                                continue;
                        /*
                         * to add pages, make sure all new pages can be
                         * allocated without receiving ENOMEM
                         */
                        INIT_LIST_HEAD(&cpu_buffer->new_pages);
                        if (__rb_allocate_pages(cpu_buffer, cpu_buffer->nr_pages_to_update,
                                                &cpu_buffer->new_pages)) {
                                /* not enough memory for new pages */
                                err = -ENOMEM;
                                goto out_err;
                        }

                        cond_resched();
                }

                cpus_read_lock();
                /*
                 * Fire off all the required work handlers
                 * We can't schedule on offline CPUs, but it's not necessary
                 * since we can change their buffer sizes without any race.
                 */
                for_each_buffer_cpu(buffer, cpu) {
                        cpu_buffer = buffer->buffers[cpu];
                        if (!cpu_buffer->nr_pages_to_update)
                                continue;

                        /* Can't run something on an offline CPU. */
                        if (!cpu_online(cpu)) {
                                rb_update_pages(cpu_buffer);
                                cpu_buffer->nr_pages_to_update = 0;
                        } else {
                                /* Run directly if possible. */
                                migrate_disable();
                                if (cpu != smp_processor_id()) {
                                        migrate_enable();
                                        schedule_work_on(cpu,
                                                         &cpu_buffer->update_pages_work);
                                } else {
                                        update_pages_handler(&cpu_buffer->update_pages_work);
                                        migrate_enable();
                                }
                        }
                }

                /* wait for all the updates to complete */
                for_each_buffer_cpu(buffer, cpu) {
                        cpu_buffer = buffer->buffers[cpu];
                        if (!cpu_buffer->nr_pages_to_update)
                                continue;

                        if (cpu_online(cpu))
                                wait_for_completion(&cpu_buffer->update_done);
                        cpu_buffer->nr_pages_to_update = 0;
                }

                cpus_read_unlock();
        } else {
                cpu_buffer = buffer->buffers[cpu_id];

                if (nr_pages == cpu_buffer->nr_pages)
                        goto out;

                /*
                 * Don't succeed if resizing is disabled, as a reader might be
                 * manipulating the ring buffer and is expecting a sane state while
                 * this is true.
                 */
                if (atomic_read(&cpu_buffer->resize_disabled)) {
                        err = -EBUSY;
                        goto out_err_unlock;
                }

                cpu_buffer->nr_pages_to_update = nr_pages -
                                                cpu_buffer->nr_pages;

                INIT_LIST_HEAD(&cpu_buffer->new_pages);
                if (cpu_buffer->nr_pages_to_update > 0 &&
                        __rb_allocate_pages(cpu_buffer, cpu_buffer->nr_pages_to_update,
                                            &cpu_buffer->new_pages)) {
                        err = -ENOMEM;
                        goto out_err;
                }

                cpus_read_lock();

                /* Can't run something on an offline CPU. */
                if (!cpu_online(cpu_id))
                        rb_update_pages(cpu_buffer);
                else {
                        /* Run directly if possible. */
                        migrate_disable();
                        if (cpu_id == smp_processor_id()) {
                                rb_update_pages(cpu_buffer);
                                migrate_enable();
                        } else {
                                migrate_enable();
                                schedule_work_on(cpu_id,
                                                 &cpu_buffer->update_pages_work);
                                wait_for_completion(&cpu_buffer->update_done);
                        }
                }

                cpu_buffer->nr_pages_to_update = 0;
                cpus_read_unlock();
        }

 out:
        /*
         * The ring buffer resize can happen with the ring buffer
         * enabled, so that the update disturbs the tracing as little
         * as possible. But if the buffer is disabled, we do not need
         * to worry about that, and we can take the time to verify
         * that the buffer is not corrupt.
         */
        if (atomic_read(&buffer->record_disabled)) {
                atomic_inc(&buffer->record_disabled);
                /*
                 * Even though the buffer was disabled, we must make sure
                 * that it is truly disabled before calling rb_check_pages.
                 * There could have been a race between checking
                 * record_disable and incrementing it.
                 */
                synchronize_rcu();
                for_each_buffer_cpu(buffer, cpu) {
                        cpu_buffer = buffer->buffers[cpu];
                        rb_check_pages(cpu_buffer);
                }
                atomic_dec(&buffer->record_disabled);
        }

        atomic_dec(&buffer->resizing);
        mutex_unlock(&buffer->mutex);
        return 0;

 out_err:
        for_each_buffer_cpu(buffer, cpu) {
                struct buffer_page *bpage, *tmp;

                cpu_buffer = buffer->buffers[cpu];
                cpu_buffer->nr_pages_to_update = 0;

                if (list_empty(&cpu_buffer->new_pages))
                        continue;

                list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,
                                        list) {
                        list_del_init(&bpage->list);
                        free_buffer_page(bpage);
                }
        }
 out_err_unlock:
        atomic_dec(&buffer->resizing);
        mutex_unlock(&buffer->mutex);
        return err;
}
EXPORT_SYMBOL_GPL(ring_buffer_resize);

void ring_buffer_change_overwrite(struct trace_buffer *buffer, int val)
{
        mutex_lock(&buffer->mutex);
        if (val)
                buffer->flags |= RB_FL_OVERWRITE;
        else
                buffer->flags &= ~RB_FL_OVERWRITE;
        mutex_unlock(&buffer->mutex);
}
EXPORT_SYMBOL_GPL(ring_buffer_change_overwrite);

static __always_inline void *__rb_page_index(struct buffer_page *bpage, unsigned index)
{
        return bpage->page->data + index;
}

static __always_inline struct ring_buffer_event *
rb_reader_event(struct ring_buffer_per_cpu *cpu_buffer)
{
        return __rb_page_index(cpu_buffer->reader_page,
                               cpu_buffer->reader_page->read);
}

static struct ring_buffer_event *
rb_iter_head_event(struct ring_buffer_iter *iter)
{
        struct ring_buffer_event *event;
        struct buffer_page *iter_head_page = iter->head_page;
        unsigned long commit;
        unsigned length;

        if (iter->head != iter->next_event)
                return iter->event;

        /*
         * When the writer goes across pages, it issues a cmpxchg which
         * is a mb(), which will synchronize with the rmb here.
         * (see rb_tail_page_update() and __rb_reserve_next())
         */
        commit = rb_page_commit(iter_head_page);
        smp_rmb();

        /* An event needs to be at least 8 bytes in size */
        if (iter->head > commit - 8)
                goto reset;

        event = __rb_page_index(iter_head_page, iter->head);
        length = rb_event_length(event);

        /*
         * READ_ONCE() doesn't work on functions and we don't want the
         * compiler doing any crazy optimizations with length.
         */
        barrier();

        if ((iter->head + length) > commit || length > iter->event_size)
                /* Writer corrupted the read? */
                goto reset;

        memcpy(iter->event, event, length);
        /*
         * If the page stamp is still the same after this rmb() then the
         * event was safely copied without the writer entering the page.
         */
        smp_rmb();

        /* Make sure the page didn't change since we read this */
        if (iter->page_stamp != iter_head_page->page->time_stamp ||
            commit > rb_page_commit(iter_head_page))
                goto reset;

        iter->next_event = iter->head + length;
        return iter->event;
 reset:
        /* Reset to the beginning */
        iter->page_stamp = iter->read_stamp = iter->head_page->page->time_stamp;
        iter->head = 0;
        iter->next_event = 0;
        iter->missed_events = 1;
        return NULL;
}

/* Size is determined by what has been committed */
static __always_inline unsigned rb_page_size(struct buffer_page *bpage)
{
        return rb_page_commit(bpage) & ~RB_MISSED_MASK;
}

static __always_inline unsigned
rb_commit_index(struct ring_buffer_per_cpu *cpu_buffer)
{
        return rb_page_commit(cpu_buffer->commit_page);
}

static __always_inline unsigned
rb_event_index(struct ring_buffer_per_cpu *cpu_buffer, struct ring_buffer_event *event)
{
        unsigned long addr = (unsigned long)event;

        addr &= (PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1;

        return addr - BUF_PAGE_HDR_SIZE;
}

static void rb_inc_iter(struct ring_buffer_iter *iter)
{
        struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;

        /*
         * The iterator could be on the reader page (it starts there).
         * But the head could have moved, since the reader was
         * found. Check for this case and assign the iterator
         * to the head page instead of next.
         */
        if (iter->head_page == cpu_buffer->reader_page)
                iter->head_page = rb_set_head_page(cpu_buffer);
        else
                rb_inc_page(&iter->head_page);

        iter->page_stamp = iter->read_stamp = iter->head_page->page->time_stamp;
        iter->head = 0;
        iter->next_event = 0;
}

/* Return the index into the sub-buffers for a given sub-buffer */
static int rb_meta_subbuf_idx(struct ring_buffer_cpu_meta *meta, void *subbuf)
{
        void *subbuf_array;

        subbuf_array = (void *)meta + sizeof(int) * meta->nr_subbufs;
        subbuf_array = (void *)ALIGN((unsigned long)subbuf_array, meta->subbuf_size);
        return (subbuf - subbuf_array) / meta->subbuf_size;
}

static void rb_update_meta_head(struct ring_buffer_per_cpu *cpu_buffer,
                                struct buffer_page *next_page)
{
        struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta;
        unsigned long old_head = (unsigned long)next_page->page;
        unsigned long new_head;

        rb_inc_page(&next_page);
        new_head = (unsigned long)next_page->page;

        /*
         * Only move it forward once, if something else came in and
         * moved it forward, then we don't want to touch it.
         */
        (void)cmpxchg(&meta->head_buffer, old_head, new_head);
}

static void rb_update_meta_reader(struct ring_buffer_per_cpu *cpu_buffer,
                                  struct buffer_page *reader)
{
        struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta;
        void *old_reader = cpu_buffer->reader_page->page;
        void *new_reader = reader->page;
        int id;

        id = reader->id;
        cpu_buffer->reader_page->id = id;
        reader->id = 0;

        meta->buffers[0] = rb_meta_subbuf_idx(meta, new_reader);
        meta->buffers[id] = rb_meta_subbuf_idx(meta, old_reader);

        /* The head pointer is the one after the reader */
        rb_update_meta_head(cpu_buffer, reader);
}

/*
 * rb_handle_head_page - writer hit the head page
 *
 * Returns: +1 to retry page
 *           0 to continue
 *          -1 on error
 */
static int
rb_handle_head_page(struct ring_buffer_per_cpu *cpu_buffer,
                    struct buffer_page *tail_page,
                    struct buffer_page *next_page)
{
        struct buffer_page *new_head;
        int entries;
        int type;
        int ret;

        entries = rb_page_entries(next_page);

        /*
         * The hard part is here. We need to move the head
         * forward, and protect against both readers on
         * other CPUs and writers coming in via interrupts.
         */
        type = rb_head_page_set_update(cpu_buffer, next_page, tail_page,
                                       RB_PAGE_HEAD);

        /*
         * type can be one of four:
         *  NORMAL - an interrupt already moved it for us
         *  HEAD   - we are the first to get here.
         *  UPDATE - we are the interrupt interrupting
         *           a current move.
         *  MOVED  - a reader on another CPU moved the next
         *           pointer to its reader page. Give up
         *           and try again.
         */

        switch (type) {
        case RB_PAGE_HEAD:
                /*
                 * We changed the head to UPDATE, thus
                 * it is our responsibility to update
                 * the counters.
                 */
                local_add(entries, &cpu_buffer->overrun);
                local_sub(rb_page_commit(next_page), &cpu_buffer->entries_bytes);
                local_inc(&cpu_buffer->pages_lost);

                if (cpu_buffer->ring_meta)
                        rb_update_meta_head(cpu_buffer, next_page);
                /*
                 * The entries will be zeroed out when we move the
                 * tail page.
                 */

                /* still more to do */
                break;

        case RB_PAGE_UPDATE:
                /*
                 * This is an interrupt that interrupt the
                 * previous update. Still more to do.
                 */
                break;
        case RB_PAGE_NORMAL:
                /*
                 * An interrupt came in before the update
                 * and processed this for us.
                 * Nothing left to do.
                 */
                return 1;
        case RB_PAGE_MOVED:
                /*
                 * The reader is on another CPU and just did
                 * a swap with our next_page.
                 * Try again.
                 */
                return 1;
        default:
                RB_WARN_ON(cpu_buffer, 1); /* WTF??? */
                return -1;
        }

        /*
         * Now that we are here, the old head pointer is
         * set to UPDATE. This will keep the reader from
         * swapping the head page with the reader page.
         * The reader (on another CPU) will spin till
         * we are finished.
         *
         * We just need to protect against interrupts
         * doing the job. We will set the next pointer
         * to HEAD. After that, we set the old pointer
         * to NORMAL, but only if it was HEAD before.
         * otherwise we are an interrupt, and only
         * want the outer most commit to reset it.
         */
        new_head = next_page;
        rb_inc_page(&new_head);

        ret = rb_head_page_set_head(cpu_buffer, new_head, next_page,
                                    RB_PAGE_NORMAL);

        /*
         * Valid returns are:
         *  HEAD   - an interrupt came in and already set it.
         *  NORMAL - One of two things:
         *            1) We really set it.
         *            2) A bunch of interrupts came in and moved
         *               the page forward again.
         */
        switch (ret) {
        case RB_PAGE_HEAD:
        case RB_PAGE_NORMAL:
                /* OK */
                break;
        default:
                RB_WARN_ON(cpu_buffer, 1);
                return -1;
        }

        /*
         * It is possible that an interrupt came in,
         * set the head up, then more interrupts came in
         * and moved it again. When we get back here,
         * the page would have been set to NORMAL but we
         * just set it back to HEAD.
         *
         * How do you detect this? Well, if that happened
         * the tail page would have moved.
         */
        if (ret == RB_PAGE_NORMAL) {
                struct buffer_page *buffer_tail_page;

                buffer_tail_page = READ_ONCE(cpu_buffer->tail_page);
                /*
                 * If the tail had moved passed next, then we need
                 * to reset the pointer.
                 */
                if (buffer_tail_page != tail_page &&
                    buffer_tail_page != next_page)
                        rb_head_page_set_normal(cpu_buffer, new_head,
                                                next_page,
                                                RB_PAGE_HEAD);
        }

        /*
         * If this was the outer most commit (the one that
         * changed the original pointer from HEAD to UPDATE),
         * then it is up to us to reset it to NORMAL.
         */
        if (type == RB_PAGE_HEAD) {
                ret = rb_head_page_set_normal(cpu_buffer, next_page,
                                              tail_page,
                                              RB_PAGE_UPDATE);
                if (RB_WARN_ON(cpu_buffer,
                               ret != RB_PAGE_UPDATE))
                        return -1;
        }

        return 0;
}

static inline void
rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,
              unsigned long tail, struct rb_event_info *info)
{
        unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size);
        struct buffer_page *tail_page = info->tail_page;
        struct ring_buffer_event *event;
        unsigned long length = info->length;

        /*
         * Only the event that crossed the page boundary
         * must fill the old tail_page with padding.
         */
        if (tail >= bsize) {
                /*
                 * If the page was filled, then we still need
                 * to update the real_end. Reset it to zero
                 * and the reader will ignore it.
                 */
                if (tail == bsize)
                        tail_page->real_end = 0;

                local_sub(length, &tail_page->write);
                return;
        }

        event = __rb_page_index(tail_page, tail);

        /*
         * Save the original length to the meta data.
         * This will be used by the reader to add lost event
         * counter.
         */
        tail_page->real_end = tail;

        /*
         * If this event is bigger than the minimum size, then
         * we need to be careful that we don't subtract the
         * write counter enough to allow another writer to slip
         * in on this page.
         * We put in a discarded commit instead, to make sure
         * that this space is not used again, and this space will
         * not be accounted into 'entries_bytes'.
         *
         * If we are less than the minimum size, we don't need to
         * worry about it.
         */
        if (tail > (bsize - RB_EVNT_MIN_SIZE)) {
                /* No room for any events */

                /* Mark the rest of the page with padding */
                rb_event_set_padding(event);

                /* Make sure the padding is visible before the write update */
                smp_wmb();

                /* Set the write back to the previous setting */
                local_sub(length, &tail_page->write);
                return;
        }

        /* Put in a discarded event */
        event->array[0] = (bsize - tail) - RB_EVNT_HDR_SIZE;
        event->type_len = RINGBUF_TYPE_PADDING;
        /* time delta must be non zero */
        event->time_delta = 1;

        /* account for padding bytes */
        local_add(bsize - tail, &cpu_buffer->entries_bytes);

        /* Make sure the padding is visible before the tail_page->write update */
        smp_wmb();

        /* Set write to end of buffer */
        length = (tail + length) - bsize;
        local_sub(length, &tail_page->write);
}

static inline void rb_end_commit(struct ring_buffer_per_cpu *cpu_buffer);

/*
 * This is the slow path, force gcc not to inline it.
 */
static noinline struct ring_buffer_event *
rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer,
             unsigned long tail, struct rb_event_info *info)
{
        struct buffer_page *tail_page = info->tail_page;
        struct buffer_page *commit_page = cpu_buffer->commit_page;
        struct trace_buffer *buffer = cpu_buffer->buffer;
        struct buffer_page *next_page;
        int ret;

        next_page = tail_page;

        rb_inc_page(&next_page);

        /*
         * If for some reason, we had an interrupt storm that made
         * it all the way around the buffer, bail, and warn
         * about it.
         */
        if (unlikely(next_page == commit_page)) {
                local_inc(&cpu_buffer->commit_overrun);
                goto out_reset;
        }

        /*
         * This is where the fun begins!
         *
         * We are fighting against races between a reader that
         * could be on another CPU trying to swap its reader
         * page with the buffer head.
         *
         * We are also fighting against interrupts coming in and
         * moving the head or tail on us as well.
         *
         * If the next page is the head page then we have filled
         * the buffer, unless the commit page is still on the
         * reader page.
         */
        if (rb_is_head_page(next_page, &tail_page->list)) {

                /*
                 * If the commit is not on the reader page, then
                 * move the header page.
                 */
                if (!rb_is_reader_page(cpu_buffer->commit_page)) {
                        /*
                         * If we are not in overwrite mode,
                         * this is easy, just stop here.
                         */
                        if (!(buffer->flags & RB_FL_OVERWRITE)) {
                                local_inc(&cpu_buffer->dropped_events);
                                goto out_reset;
                        }

                        ret = rb_handle_head_page(cpu_buffer,
                                                  tail_page,
                                                  next_page);
                        if (ret < 0)
                                goto out_reset;
                        if (ret)
                                goto out_again;
                } else {
                        /*
                         * We need to be careful here too. The
                         * commit page could still be on the reader
                         * page. We could have a small buffer, and
                         * have filled up the buffer with events
                         * from interrupts and such, and wrapped.
                         *
                         * Note, if the tail page is also on the
                         * reader_page, we let it move out.
                         */
                        if (unlikely((cpu_buffer->commit_page !=
                                      cpu_buffer->tail_page) &&
                                     (cpu_buffer->commit_page ==
                                      cpu_buffer->reader_page))) {
                                local_inc(&cpu_buffer->commit_overrun);
                                goto out_reset;
                        }
                }
        }

        rb_tail_page_update(cpu_buffer, tail_page, next_page);

 out_again:

        rb_reset_tail(cpu_buffer, tail, info);

        /* Commit what we have for now. */
        rb_end_commit(cpu_buffer);
        /* rb_end_commit() decs committing */
        local_inc(&cpu_buffer->committing);

        /* fail and let the caller try again */
        return ERR_PTR(-EAGAIN);

 out_reset:
        /* reset write */
        rb_reset_tail(cpu_buffer, tail, info);

        return NULL;
}

/* Slow path */
static struct ring_buffer_event *
rb_add_time_stamp(struct ring_buffer_per_cpu *cpu_buffer,
                  struct ring_buffer_event *event, u64 delta, bool abs)
{
        if (abs)
                event->type_len = RINGBUF_TYPE_TIME_STAMP;
        else
                event->type_len = RINGBUF_TYPE_TIME_EXTEND;

        /* Not the first event on the page, or not delta? */
        if (abs || rb_event_index(cpu_buffer, event)) {
                event->time_delta = delta & TS_MASK;
                event->array[0] = delta >> TS_SHIFT;
        } else {
                /* nope, just zero it */
                event->time_delta = 0;
                event->array[0] = 0;
        }

        return skip_time_extend(event);
}

#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
static inline bool sched_clock_stable(void)
{
        return true;
}
#endif

static void
rb_check_timestamp(struct ring_buffer_per_cpu *cpu_buffer,
                   struct rb_event_info *info)
{
        u64 write_stamp;

        WARN_ONCE(1, "Delta way too big! %llu ts=%llu before=%llu after=%llu write stamp=%llu\n%s",
                  (unsigned long long)info->delta,
                  (unsigned long long)info->ts,
                  (unsigned long long)info->before,
                  (unsigned long long)info->after,
                  (unsigned long long)({rb_time_read(&cpu_buffer->write_stamp, &write_stamp); write_stamp;}),
                  sched_clock_stable() ? "" :
                  "If you just came from a suspend/resume,\n"
                  "please switch to the trace global clock:\n"
                  "  echo global > /sys/kernel/tracing/trace_clock\n"
                  "or add trace_clock=global to the kernel command line\n");
}

static void rb_add_timestamp(struct ring_buffer_per_cpu *cpu_buffer,
                                      struct ring_buffer_event **event,
                                      struct rb_event_info *info,
                                      u64 *delta,
                                      unsigned int *length)
{
        bool abs = info->add_timestamp &
                (RB_ADD_STAMP_FORCE | RB_ADD_STAMP_ABSOLUTE);

        if (unlikely(info->delta > (1ULL << 59))) {
                /*
                 * Some timers can use more than 59 bits, and when a timestamp
                 * is added to the buffer, it will lose those bits.
                 */
                if (abs && (info->ts & TS_MSB)) {
                        info->delta &= ABS_TS_MASK;

                /* did the clock go backwards */
                } else if (info->before == info->after && info->before > info->ts) {
                        /* not interrupted */
                        static int once;

                        /*
                         * This is possible with a recalibrating of the TSC.
                         * Do not produce a call stack, but just report it.
                         */
                        if (!once) {
                                once++;
                                pr_warn("Ring buffer clock went backwards: %llu -> %llu\n",
                                        info->before, info->ts);
                        }
                } else
                        rb_check_timestamp(cpu_buffer, info);
                if (!abs)
                        info->delta = 0;
        }
        *event = rb_add_time_stamp(cpu_buffer, *event, info->delta, abs);
        *length -= RB_LEN_TIME_EXTEND;
        *delta = 0;
}

/**
 * rb_update_event - update event type and data
 * @cpu_buffer: The per cpu buffer of the @event
 * @event: the event to update
 * @info: The info to update the @event with (contains length and delta)
 *
 * Update the type and data fields of the @event. The length
 * is the actual size that is written to the ring buffer,
 * and with this, we can determine what to place into the
 * data field.
 */
static void
rb_update_event(struct ring_buffer_per_cpu *cpu_buffer,
                struct ring_buffer_event *event,
                struct rb_event_info *info)
{
        unsigned length = info->length;
        u64 delta = info->delta;
        unsigned int nest = local_read(&cpu_buffer->committing) - 1;

        if (!WARN_ON_ONCE(nest >= MAX_NEST))
                cpu_buffer->event_stamp[nest] = info->ts;

        /*
         * If we need to add a timestamp, then we
         * add it to the start of the reserved space.
         */
        if (unlikely(info->add_timestamp))
                rb_add_timestamp(cpu_buffer, &event, info, &delta, &length);

        event->time_delta = delta;
        length -= RB_EVNT_HDR_SIZE;
        if (length > RB_MAX_SMALL_DATA || RB_FORCE_8BYTE_ALIGNMENT) {
                event->type_len = 0;
                event->array[0] = length;
        } else
                event->type_len = DIV_ROUND_UP(length, RB_ALIGNMENT);
}

static unsigned rb_calculate_event_length(unsigned length)
{
        struct ring_buffer_event event; /* Used only for sizeof array */

        /* zero length can cause confusions */
        if (!length)
                length++;

        if (length > RB_MAX_SMALL_DATA || RB_FORCE_8BYTE_ALIGNMENT)
                length += sizeof(event.array[0]);

        length += RB_EVNT_HDR_SIZE;
        length = ALIGN(length, RB_ARCH_ALIGNMENT);

        /*
         * In case the time delta is larger than the 27 bits for it
         * in the header, we need to add a timestamp. If another
         * event comes in when trying to discard this one to increase
         * the length, then the timestamp will be added in the allocated
         * space of this event. If length is bigger than the size needed
         * for the TIME_EXTEND, then padding has to be used. The events
         * length must be either RB_LEN_TIME_EXTEND, or greater than or equal
         * to RB_LEN_TIME_EXTEND + 8, as 8 is the minimum size for padding.
         * As length is a multiple of 4, we only need to worry if it
         * is 12 (RB_LEN_TIME_EXTEND + 4).
         */
        if (length == RB_LEN_TIME_EXTEND + RB_ALIGNMENT)
                length += RB_ALIGNMENT;

        return length;
}

static inline bool
rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer,
                  struct ring_buffer_event *event)
{
        unsigned long new_index, old_index;
        struct buffer_page *bpage;
        unsigned long addr;

        new_index = rb_event_index(cpu_buffer, event);
        old_index = new_index + rb_event_ts_length(event);
        addr = (unsigned long)event;
        addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1);

        bpage = READ_ONCE(cpu_buffer->tail_page);

        /*
         * Make sure the tail_page is still the same and
         * the next write location is the end of this event
         */
        if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) {
                unsigned long write_mask =
                        local_read(&bpage->write) & ~RB_WRITE_MASK;
                unsigned long event_length = rb_event_length(event);

                /*
                 * For the before_stamp to be different than the write_stamp
                 * to make sure that the next event adds an absolute
                 * value and does not rely on the saved write stamp, which
                 * is now going to be bogus.
                 *
                 * By setting the before_stamp to zero, the next event
                 * is not going to use the write_stamp and will instead
                 * create an absolute timestamp. This means there's no
                 * reason to update the wirte_stamp!
                 */
                rb_time_set(&cpu_buffer->before_stamp, 0);

                /*
                 * If an event were to come in now, it would see that the
                 * write_stamp and the before_stamp are different, and assume
                 * that this event just added itself before updating
                 * the write stamp. The interrupting event will fix the
                 * write stamp for us, and use an absolute timestamp.
                 */

                /*
                 * This is on the tail page. It is possible that
                 * a write could come in and move the tail page
                 * and write to the next page. That is fine
                 * because we just shorten what is on this page.
                 */
                old_index += write_mask;
                new_index += write_mask;

                /* caution: old_index gets updated on cmpxchg failure */
                if (local_try_cmpxchg(&bpage->write, &old_index, new_index)) {
                        /* update counters */
                        local_sub(event_length, &cpu_buffer->entries_bytes);
                        return true;
                }
        }

        /* could not discard */
        return false;
}

static void rb_start_commit(struct ring_buffer_per_cpu *cpu_buffer)
{
        local_inc(&cpu_buffer->committing);
        local_inc(&cpu_buffer->commits);
}

static __always_inline void
rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
{
        unsigned long max_count;

        /*
         * We only race with interrupts and NMIs on this CPU.
         * If we own the commit event, then we can commit
         * all others that interrupted us, since the interruptions
         * are in stack format (they finish before they come
         * back to us). This allows us to do a simple loop to
         * assign the commit to the tail.
         */
 again:
        max_count = cpu_buffer->nr_pages * 100;

        while (cpu_buffer->commit_page != READ_ONCE(cpu_buffer->tail_page)) {
                if (RB_WARN_ON(cpu_buffer, !(--max_count)))
                        return;
                if (RB_WARN_ON(cpu_buffer,
                               rb_is_reader_page(cpu_buffer->tail_page)))
                        return;
                /*
                 * No need for a memory barrier here, as the update
                 * of the tail_page did it for this page.
                 */
                local_set(&cpu_buffer->commit_page->page->commit,
                          rb_page_write(cpu_buffer->commit_page));
                rb_inc_page(&cpu_buffer->commit_page);
                if (cpu_buffer->ring_meta) {
                        struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta;
                        meta->commit_buffer = (unsigned long)cpu_buffer->commit_page->page;
                }
                /* add barrier to keep gcc from optimizing too much */
                barrier();
        }
        while (rb_commit_index(cpu_buffer) !=
               rb_page_write(cpu_buffer->commit_page)) {

                /* Make sure the readers see the content of what is committed. */
                smp_wmb();
                local_set(&cpu_buffer->commit_page->page->commit,
                          rb_page_write(cpu_buffer->commit_page));
                RB_WARN_ON(cpu_buffer,
                           local_read(&cpu_buffer->commit_page->page->commit) &
                           ~RB_WRITE_MASK);
                barrier();
        }

        /* again, keep gcc from optimizing */
        barrier();

        /*
         * If an interrupt came in just after the first while loop
         * and pushed the tail page forward, we will be left with
         * a dangling commit that will never go forward.
         */
        if (unlikely(cpu_buffer->commit_page != READ_ONCE(cpu_buffer->tail_page)))
                goto again;
}

static __always_inline void rb_end_commit(struct ring_buffer_per_cpu *cpu_buffer)
{
        unsigned long commits;

        if (RB_WARN_ON(cpu_buffer,
                       !local_read(&cpu_buffer->committing)))
                return;

 again:
        commits = local_read(&cpu_buffer->commits);
        /* synchronize with interrupts */
        barrier();
        if (local_read(&cpu_buffer->committing) == 1)
                rb_set_commit_to_write(cpu_buffer);

        local_dec(&cpu_buffer->committing);

        /* synchronize with interrupts */
        barrier();

        /*
         * Need to account for interrupts coming in between the
         * updating of the commit page and the clearing of the
         * committing counter.
         */
        if (unlikely(local_read(&cpu_buffer->commits) != commits) &&
            !local_read(&cpu_buffer->committing)) {
                local_inc(&cpu_buffer->committing);
                goto again;
        }
}

static inline void rb_event_discard(struct ring_buffer_event *event)
{
        if (extended_time(event))
                event = skip_time_extend(event);

        /* array[0] holds the actual length for the discarded event */
        event->array[0] = rb_event_data_length(event) - RB_EVNT_HDR_SIZE;
        event->type_len = RINGBUF_TYPE_PADDING;
        /* time delta must be non zero */
        if (!event->time_delta)
                event->time_delta = 1;
}

static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer)
{
        local_inc(&cpu_buffer->entries);
        rb_end_commit(cpu_buffer);
}

static __always_inline void
rb_wakeups(struct trace_buffer *buffer, struct ring_buffer_per_cpu *cpu_buffer)
{
        if (buffer->irq_work.waiters_pending) {
                buffer->irq_work.waiters_pending = false;
                /* irq_work_queue() supplies it's own memory barriers */
                irq_work_queue(&buffer->irq_work.work);
        }

        if (cpu_buffer->irq_work.waiters_pending) {
                cpu_buffer->irq_work.waiters_pending = false;
                /* irq_work_queue() supplies it's own memory barriers */
                irq_work_queue(&cpu_buffer->irq_work.work);
        }

        if (cpu_buffer->last_pages_touch == local_read(&cpu_buffer->pages_touched))
                return;

        if (cpu_buffer->reader_page == cpu_buffer->commit_page)
                return;

        if (!cpu_buffer->irq_work.full_waiters_pending)
                return;

        cpu_buffer->last_pages_touch = local_read(&cpu_buffer->pages_touched);

        if (!full_hit(buffer, cpu_buffer->cpu, cpu_buffer->shortest_full))
                return;

        cpu_buffer->irq_work.wakeup_full = true;
        cpu_buffer->irq_work.full_waiters_pending = false;
        /* irq_work_queue() supplies it's own memory barriers */
        irq_work_queue(&cpu_buffer->irq_work.work);
}

#ifdef CONFIG_RING_BUFFER_RECORD_RECURSION
# define do_ring_buffer_record_recursion()        \
        do_ftrace_record_recursion(_THIS_IP_, _RET_IP_)
#else
# define do_ring_buffer_record_recursion() do { } while (0)
#endif

/*
 * The lock and unlock are done within a preempt disable section.
 * The current_context per_cpu variable can only be modified
 * by the current task between lock and unlock. But it can
 * be modified more than once via an interrupt. To pass this
 * information from the lock to the unlock without having to
 * access the 'in_interrupt()' functions again (which do show
 * a bit of overhead in something as critical as function tracing,
 * we use a bitmask trick.
 *
 *  bit 1 =  NMI context
 *  bit 2 =  IRQ context
 *  bit 3 =  SoftIRQ context
 *  bit 4 =  normal context.
 *
 * This works because this is the order of contexts that can
 * preempt other contexts. A SoftIRQ never preempts an IRQ
 * context.
 *
 * When the context is determined, the corresponding bit is
 * checked and set (if it was set, then a recursion of that context
 * happened).
 *
 * On unlock, we need to clear this bit. To do so, just subtract
 * 1 from the current_context and AND it to itself.
 *
 * (binary)
 *  101 - 1 = 100
 *  101 & 100 = 100 (clearing bit zero)
 *
 *  1010 - 1 = 1001
 *  1010 & 1001 = 1000 (clearing bit 1)
 *
 * The least significant bit can be cleared this way, and it
 * just so happens that it is the same bit corresponding to
 * the current context.
 *
 * Now the TRANSITION bit breaks the above slightly. The TRANSITION bit
 * is set when a recursion is detected at the current context, and if
 * the TRANSITION bit is already set, it will fail the recursion.
 * This is needed because there's a lag between the changing of
 * interrupt context and updating the preempt count. In this case,
 * a false positive will be found. To handle this, one extra recursion
 * is allowed, and this is done by the TRANSITION bit. If the TRANSITION
 * bit is already set, then it is considered a recursion and the function
 * ends. Otherwise, the TRANSITION bit is set, and that bit is returned.
 *
 * On the trace_recursive_unlock(), the TRANSITION bit will be the first
 * to be cleared. Even if it wasn't the context that set it. That is,
 * if an interrupt comes in while NORMAL bit is set and the ring buffer
 * is called before preempt_count() is updated, since the check will
 * be on the NORMAL bit, the TRANSITION bit will then be set. If an
 * NMI then comes in, it will set the NMI bit, but when the NMI code
 * does the trace_recursive_unlock() it will clear the TRANSITION bit
 * and leave the NMI bit set. But this is fine, because the interrupt
 * code that set the TRANSITION bit will then clear the NMI bit when it
 * calls trace_recursive_unlock(). If another NMI comes in, it will
 * set the TRANSITION bit and continue.
 *
 * Note: The TRANSITION bit only handles a single transition between context.
 */

static __always_inline bool
trace_recursive_lock(struct ring_buffer_per_cpu *cpu_buffer)
{
        unsigned int val = cpu_buffer->current_context;
        int bit = interrupt_context_level();

        bit = RB_CTX_NORMAL - bit;

        if (unlikely(val & (1 << (bit + cpu_buffer->nest)))) {
                /*
                 * It is possible that this was called by transitioning
                 * between interrupt context, and preempt_count() has not
                 * been updated yet. In this case, use the TRANSITION bit.
                 */
                bit = RB_CTX_TRANSITION;
                if (val & (1 << (bit + cpu_buffer->nest))) {
                        do_ring_buffer_record_recursion();
                        return true;
                }
        }

        val |= (1 << (bit + cpu_buffer->nest));
        cpu_buffer->current_context = val;

        return false;
}

static __always_inline void
trace_recursive_unlock(struct ring_buffer_per_cpu *cpu_buffer)
{
        cpu_buffer->current_context &=
                cpu_buffer->current_context - (1 << cpu_buffer->nest);
}

/* The recursive locking above uses 5 bits */
#define NESTED_BITS 5

/**
 * ring_buffer_nest_start - Allow to trace while nested
 * @buffer: The ring buffer to modify
 *
 * The ring buffer has a safety mechanism to prevent recursion.
 * But there may be a case where a trace needs to be done while
 * tracing something else. In this case, calling this function
 * will allow this function to nest within a currently active
 * ring_buffer_lock_reserve().
 *
 * Call this function before calling another ring_buffer_lock_reserve() and
 * call ring_buffer_nest_end() after the nested ring_buffer_unlock_commit().
 */
void ring_buffer_nest_start(struct trace_buffer *buffer)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        int cpu;

        /* Enabled by ring_buffer_nest_end() */
        preempt_disable_notrace();
        cpu = raw_smp_processor_id();
        cpu_buffer = buffer->buffers[cpu];
        /* This is the shift value for the above recursive locking */
        cpu_buffer->nest += NESTED_BITS;
}

/**
 * ring_buffer_nest_end - Allow to trace while nested
 * @buffer: The ring buffer to modify
 *
 * Must be called after ring_buffer_nest_start() and after the
 * ring_buffer_unlock_commit().
 */
void ring_buffer_nest_end(struct trace_buffer *buffer)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        int cpu;

        /* disabled by ring_buffer_nest_start() */
        cpu = raw_smp_processor_id();
        cpu_buffer = buffer->buffers[cpu];
        /* This is the shift value for the above recursive locking */
        cpu_buffer->nest -= NESTED_BITS;
        preempt_enable_notrace();
}

/**
 * ring_buffer_unlock_commit - commit a reserved
 * @buffer: The buffer to commit to
 *
 * This commits the data to the ring buffer, and releases any locks held.
 *
 * Must be paired with ring_buffer_lock_reserve.
 */
int ring_buffer_unlock_commit(struct trace_buffer *buffer)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        int cpu = raw_smp_processor_id();

        cpu_buffer = buffer->buffers[cpu];

        rb_commit(cpu_buffer);

        rb_wakeups(buffer, cpu_buffer);

        trace_recursive_unlock(cpu_buffer);

        preempt_enable_notrace();

        return 0;
}
EXPORT_SYMBOL_GPL(ring_buffer_unlock_commit);

/* Special value to validate all deltas on a page. */
#define CHECK_FULL_PAGE                1L

#ifdef CONFIG_RING_BUFFER_VALIDATE_TIME_DELTAS

static const char *show_irq_str(int bits)
{
        const char *type[] = {
                ".",        // 0
                "s",        // 1
                "h",        // 2
                "Hs",        // 3
                "n",        // 4
                "Ns",        // 5
                "Nh",        // 6
                "NHs",        // 7
        };

        return type[bits];
}

/* Assume this is a trace event */
static const char *show_flags(struct ring_buffer_event *event)
{
        struct trace_entry *entry;
        int bits = 0;

        if (rb_event_data_length(event) - RB_EVNT_HDR_SIZE < sizeof(*entry))
                return "X";

        entry = ring_buffer_event_data(event);

        if (entry->flags & TRACE_FLAG_SOFTIRQ)
                bits |= 1;

        if (entry->flags & TRACE_FLAG_HARDIRQ)
                bits |= 2;

        if (entry->flags & TRACE_FLAG_NMI)
                bits |= 4;

        return show_irq_str(bits);
}

static const char *show_irq(struct ring_buffer_event *event)
{
        struct trace_entry *entry;

        if (rb_event_data_length(event) - RB_EVNT_HDR_SIZE < sizeof(*entry))
                return "";

        entry = ring_buffer_event_data(event);
        if (entry->flags & TRACE_FLAG_IRQS_OFF)
                return "d";
        return "";
}

static const char *show_interrupt_level(void)
{
        unsigned long pc = preempt_count();
        unsigned char level = 0;

        if (pc & SOFTIRQ_OFFSET)
                level |= 1;

        if (pc & HARDIRQ_MASK)
                level |= 2;

        if (pc & NMI_MASK)
                level |= 4;

        return show_irq_str(level);
}

static void dump_buffer_page(struct buffer_data_page *bpage,
                             struct rb_event_info *info,
                             unsigned long tail)
{
        struct ring_buffer_event *event;
        u64 ts, delta;
        int e;

        ts = bpage->time_stamp;
        pr_warn("  [%lld] PAGE TIME STAMP\n", ts);

        for (e = 0; e < tail; e += rb_event_length(event)) {

                event = (struct ring_buffer_event *)(bpage->data + e);

                switch (event->type_len) {

                case RINGBUF_TYPE_TIME_EXTEND:
                        delta = rb_event_time_stamp(event);
                        ts += delta;
                        pr_warn(" 0x%x: [%lld] delta:%lld TIME EXTEND\n",
                                e, ts, delta);
                        break;

                case RINGBUF_TYPE_TIME_STAMP:
                        delta = rb_event_time_stamp(event);
                        ts = rb_fix_abs_ts(delta, ts);
                        pr_warn(" 0x%x:  [%lld] absolute:%lld TIME STAMP\n",
                                e, ts, delta);
                        break;

                case RINGBUF_TYPE_PADDING:
                        ts += event->time_delta;
                        pr_warn(" 0x%x:  [%lld] delta:%d PADDING\n",
                                e, ts, event->time_delta);
                        break;

                case RINGBUF_TYPE_DATA:
                        ts += event->time_delta;
                        pr_warn(" 0x%x:  [%lld] delta:%d %s%s\n",
                                e, ts, event->time_delta,
                                show_flags(event), show_irq(event));
                        break;

                default:
                        break;
                }
        }
        pr_warn("expected end:0x%lx last event actually ended at:0x%x\n", tail, e);
}

static DEFINE_PER_CPU(atomic_t, checking);
static atomic_t ts_dump;

#define buffer_warn_return(fmt, ...)                                        \
        do {                                                                \
                /* If another report is happening, ignore this one */        \
                if (atomic_inc_return(&ts_dump) != 1) {                        \
                        atomic_dec(&ts_dump);                                \
                        goto out;                                        \
                }                                                        \
                atomic_inc(&cpu_buffer->record_disabled);                \
                pr_warn(fmt, ##__VA_ARGS__);                                \
                dump_buffer_page(bpage, info, tail);                        \
                atomic_dec(&ts_dump);                                        \
                /* There's some cases in boot up that this can happen */ \
                if (WARN_ON_ONCE(system_state != SYSTEM_BOOTING))        \
                        /* Do not re-enable checking */                        \
                        return;                                                \
        } while (0)

/*
 * Check if the current event time stamp matches the deltas on
 * the buffer page.
 */
static void check_buffer(struct ring_buffer_per_cpu *cpu_buffer,
                         struct rb_event_info *info,
                         unsigned long tail)
{
        struct buffer_data_page *bpage;
        u64 ts, delta;
        bool full = false;
        int ret;

        bpage = info->tail_page->page;

        if (tail == CHECK_FULL_PAGE) {
                full = true;
                tail = local_read(&bpage->commit);
        } else if (info->add_timestamp &
                   (RB_ADD_STAMP_FORCE | RB_ADD_STAMP_ABSOLUTE)) {
                /* Ignore events with absolute time stamps */
                return;
        }

        /*
         * Do not check the first event (skip possible extends too).
         * Also do not check if previous events have not been committed.
         */
        if (tail <= 8 || tail > local_read(&bpage->commit))
                return;

        /*
         * If this interrupted another event,
         */
        if (atomic_inc_return(this_cpu_ptr(&checking)) != 1)
                goto out;

        ret = rb_read_data_buffer(bpage, tail, cpu_buffer->cpu, &ts, &delta);
        if (ret < 0) {
                if (delta < ts) {
                        buffer_warn_return("[CPU: %d]ABSOLUTE TIME WENT BACKWARDS: last ts: %lld absolute ts: %lld\n",
                                           cpu_buffer->cpu, ts, delta);
                        goto out;
                }
        }
        if ((full && ts > info->ts) ||
            (!full && ts + info->delta != info->ts)) {
                buffer_warn_return("[CPU: %d]TIME DOES NOT MATCH expected:%lld actual:%lld delta:%lld before:%lld after:%lld%s context:%s\n",
                                   cpu_buffer->cpu,
                                   ts + info->delta, info->ts, info->delta,
                                   info->before, info->after,
                                   full ? " (full)" : "", show_interrupt_level());
        }
out:
        atomic_dec(this_cpu_ptr(&checking));
}
#else
static inline void check_buffer(struct ring_buffer_per_cpu *cpu_buffer,
                         struct rb_event_info *info,
                         unsigned long tail)
{
}
#endif /* CONFIG_RING_BUFFER_VALIDATE_TIME_DELTAS */

static struct ring_buffer_event *
__rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
                  struct rb_event_info *info)
{
        struct ring_buffer_event *event;
        struct buffer_page *tail_page;
        unsigned long tail, write, w;

        /* Don't let the compiler play games with cpu_buffer->tail_page */
        tail_page = info->tail_page = READ_ONCE(cpu_buffer->tail_page);

 /*A*/        w = local_read(&tail_page->write) & RB_WRITE_MASK;
        barrier();
        rb_time_read(&cpu_buffer->before_stamp, &info->before);
        rb_time_read(&cpu_buffer->write_stamp, &info->after);
        barrier();
        info->ts = rb_time_stamp(cpu_buffer->buffer);

        if ((info->add_timestamp & RB_ADD_STAMP_ABSOLUTE)) {
                info->delta = info->ts;
        } else {
                /*
                 * If interrupting an event time update, we may need an
                 * absolute timestamp.
                 * Don't bother if this is the start of a new page (w == 0).
                 */
                if (!w) {
                        /* Use the sub-buffer timestamp */
                        info->delta = 0;
                } else if (unlikely(info->before != info->after)) {
                        info->add_timestamp |= RB_ADD_STAMP_FORCE | RB_ADD_STAMP_EXTEND;
                        info->length += RB_LEN_TIME_EXTEND;
                } else {
                        info->delta = info->ts - info->after;
                        if (unlikely(test_time_stamp(info->delta))) {
                                info->add_timestamp |= RB_ADD_STAMP_EXTEND;
                                info->length += RB_LEN_TIME_EXTEND;
                        }
                }
        }

 /*B*/        rb_time_set(&cpu_buffer->before_stamp, info->ts);

 /*C*/        write = local_add_return(info->length, &tail_page->write);

        /* set write to only the index of the write */
        write &= RB_WRITE_MASK;

        tail = write - info->length;

        /* See if we shot pass the end of this buffer page */
        if (unlikely(write > cpu_buffer->buffer->subbuf_size)) {
                check_buffer(cpu_buffer, info, CHECK_FULL_PAGE);
                return rb_move_tail(cpu_buffer, tail, info);
        }

        if (likely(tail == w)) {
                /* Nothing interrupted us between A and C */
 /*D*/                rb_time_set(&cpu_buffer->write_stamp, info->ts);
                /*
                 * If something came in between C and D, the write stamp
                 * may now not be in sync. But that's fine as the before_stamp
                 * will be different and then next event will just be forced
                 * to use an absolute timestamp.
                 */
                if (likely(!(info->add_timestamp &
                             (RB_ADD_STAMP_FORCE | RB_ADD_STAMP_ABSOLUTE))))
                        /* This did not interrupt any time update */
                        info->delta = info->ts - info->after;
                else
                        /* Just use full timestamp for interrupting event */
                        info->delta = info->ts;
                check_buffer(cpu_buffer, info, tail);
        } else {
                u64 ts;
                /* SLOW PATH - Interrupted between A and C */

                /* Save the old before_stamp */
                rb_time_read(&cpu_buffer->before_stamp, &info->before);

                /*
                 * Read a new timestamp and update the before_stamp to make
                 * the next event after this one force using an absolute
                 * timestamp. This is in case an interrupt were to come in
                 * between E and F.
                 */
                ts = rb_time_stamp(cpu_buffer->buffer);
                rb_time_set(&cpu_buffer->before_stamp, ts);

                barrier();
 /*E*/                rb_time_read(&cpu_buffer->write_stamp, &info->after);
                barrier();
 /*F*/                if (write == (local_read(&tail_page->write) & RB_WRITE_MASK) &&
                    info->after == info->before && info->after < ts) {
                        /*
                         * Nothing came after this event between C and F, it is
                         * safe to use info->after for the delta as it
                         * matched info->before and is still valid.
                         */
                        info->delta = ts - info->after;
                } else {
                        /*
                         * Interrupted between C and F:
                         * Lost the previous events time stamp. Just set the
                         * delta to zero, and this will be the same time as
                         * the event this event interrupted. And the events that
                         * came after this will still be correct (as they would
                         * have built their delta on the previous event.
                         */
                        info->delta = 0;
                }
                info->ts = ts;
                info->add_timestamp &= ~RB_ADD_STAMP_FORCE;
        }

        /*
         * If this is the first commit on the page, then it has the same
         * timestamp as the page itself.
         */
        if (unlikely(!tail && !(info->add_timestamp &
                                (RB_ADD_STAMP_FORCE | RB_ADD_STAMP_ABSOLUTE))))
                info->delta = 0;

        /* We reserved something on the buffer */

        event = __rb_page_index(tail_page, tail);
        rb_update_event(cpu_buffer, event, info);

        local_inc(&tail_page->entries);

        /*
         * If this is the first commit on the page, then update
         * its timestamp.
         */
        if (unlikely(!tail))
                tail_page->page->time_stamp = info->ts;

        /* account for these added bytes */
        local_add(info->length, &cpu_buffer->entries_bytes);

        return event;
}

static __always_inline struct ring_buffer_event *
rb_reserve_next_event(struct trace_buffer *buffer,
                      struct ring_buffer_per_cpu *cpu_buffer,
                      unsigned long length)
{
        struct ring_buffer_event *event;
        struct rb_event_info info;
        int nr_loops = 0;
        int add_ts_default;

        /*
         * ring buffer does cmpxchg as well as atomic64 operations
         * (which some archs use locking for atomic64), make sure this
         * is safe in NMI context
         */
        if ((!IS_ENABLED(CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG) ||
             IS_ENABLED(CONFIG_GENERIC_ATOMIC64)) &&
            (unlikely(in_nmi()))) {
                return NULL;
        }

        rb_start_commit(cpu_buffer);
        /* The commit page can not change after this */

#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
        /*
         * Due to the ability to swap a cpu buffer from a buffer
         * it is possible it was swapped before we committed.
         * (committing stops a swap). We check for it here and
         * if it happened, we have to fail the write.
         */
        barrier();
        if (unlikely(READ_ONCE(cpu_buffer->buffer) != buffer)) {
                local_dec(&cpu_buffer->committing);
                local_dec(&cpu_buffer->commits);
                return NULL;
        }
#endif

        info.length = rb_calculate_event_length(length);

        if (ring_buffer_time_stamp_abs(cpu_buffer->buffer)) {
                add_ts_default = RB_ADD_STAMP_ABSOLUTE;
                info.length += RB_LEN_TIME_EXTEND;
                if (info.length > cpu_buffer->buffer->max_data_size)
                        goto out_fail;
        } else {
                add_ts_default = RB_ADD_STAMP_NONE;
        }

 again:
        info.add_timestamp = add_ts_default;
        info.delta = 0;

        /*
         * We allow for interrupts to reenter here and do a trace.
         * If one does, it will cause this original code to loop
         * back here. Even with heavy interrupts happening, this
         * should only happen a few times in a row. If this happens
         * 1000 times in a row, there must be either an interrupt
         * storm or we have something buggy.
         * Bail!
         */
        if (RB_WARN_ON(cpu_buffer, ++nr_loops > 1000))
                goto out_fail;

        event = __rb_reserve_next(cpu_buffer, &info);

        if (unlikely(PTR_ERR(event) == -EAGAIN)) {
                if (info.add_timestamp & (RB_ADD_STAMP_FORCE | RB_ADD_STAMP_EXTEND))
                        info.length -= RB_LEN_TIME_EXTEND;
                goto again;
        }

        if (likely(event))
                return event;
 out_fail:
        rb_end_commit(cpu_buffer);
        return NULL;
}

/**
 * ring_buffer_lock_reserve - reserve a part of the buffer
 * @buffer: the ring buffer to reserve from
 * @length: the length of the data to reserve (excluding event header)
 *
 * Returns a reserved event on the ring buffer to copy directly to.
 * The user of this interface will need to get the body to write into
 * and can use the ring_buffer_event_data() interface.
 *
 * The length is the length of the data needed, not the event length
 * which also includes the event header.
 *
 * Must be paired with ring_buffer_unlock_commit, unless NULL is returned.
 * If NULL is returned, then nothing has been allocated or locked.
 */
struct ring_buffer_event *
ring_buffer_lock_reserve(struct trace_buffer *buffer, unsigned long length)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        struct ring_buffer_event *event;
        int cpu;

        /* If we are tracing schedule, we don't want to recurse */
        preempt_disable_notrace();

        if (unlikely(atomic_read(&buffer->record_disabled)))
                goto out;

        cpu = raw_smp_processor_id();

        if (unlikely(!cpumask_test_cpu(cpu, buffer->cpumask)))
                goto out;

        cpu_buffer = buffer->buffers[cpu];

        if (unlikely(atomic_read(&cpu_buffer->record_disabled)))
                goto out;

        if (unlikely(length > buffer->max_data_size))
                goto out;

        if (unlikely(trace_recursive_lock(cpu_buffer)))
                goto out;

        event = rb_reserve_next_event(buffer, cpu_buffer, length);
        if (!event)
                goto out_unlock;

        return event;

 out_unlock:
        trace_recursive_unlock(cpu_buffer);
 out:
        preempt_enable_notrace();
        return NULL;
}
EXPORT_SYMBOL_GPL(ring_buffer_lock_reserve);

/*
 * Decrement the entries to the page that an event is on.
 * The event does not even need to exist, only the pointer
 * to the page it is on. This may only be called before the commit
 * takes place.
 */
static inline void
rb_decrement_entry(struct ring_buffer_per_cpu *cpu_buffer,
                   struct ring_buffer_event *event)
{
        unsigned long addr = (unsigned long)event;
        struct buffer_page *bpage = cpu_buffer->commit_page;
        struct buffer_page *start;

        addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1);

        /* Do the likely case first */
        if (likely(bpage->page == (void *)addr)) {
                local_dec(&bpage->entries);
                return;
        }

        /*
         * Because the commit page may be on the reader page we
         * start with the next page and check the end loop there.
         */
        rb_inc_page(&bpage);
        start = bpage;
        do {
                if (bpage->page == (void *)addr) {
                        local_dec(&bpage->entries);
                        return;
                }
                rb_inc_page(&bpage);
        } while (bpage != start);

        /* commit not part of this buffer?? */
        RB_WARN_ON(cpu_buffer, 1);
}

/**
 * ring_buffer_discard_commit - discard an event that has not been committed
 * @buffer: the ring buffer
 * @event: non committed event to discard
 *
 * Sometimes an event that is in the ring buffer needs to be ignored.
 * This function lets the user discard an event in the ring buffer
 * and then that event will not be read later.
 *
 * This function only works if it is called before the item has been
 * committed. It will try to free the event from the ring buffer
 * if another event has not been added behind it.
 *
 * If another event has been added behind it, it will set the event
 * up as discarded, and perform the commit.
 *
 * If this function is called, do not call ring_buffer_unlock_commit on
 * the event.
 */
void ring_buffer_discard_commit(struct trace_buffer *buffer,
                                struct ring_buffer_event *event)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        int cpu;

        /* The event is discarded regardless */
        rb_event_discard(event);

        cpu = smp_processor_id();
        cpu_buffer = buffer->buffers[cpu];

        /*
         * This must only be called if the event has not been
         * committed yet. Thus we can assume that preemption
         * is still disabled.
         */
        RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing));

        rb_decrement_entry(cpu_buffer, event);
        if (rb_try_to_discard(cpu_buffer, event))
                goto out;

 out:
        rb_end_commit(cpu_buffer);

        trace_recursive_unlock(cpu_buffer);

        preempt_enable_notrace();

}
EXPORT_SYMBOL_GPL(ring_buffer_discard_commit);

/**
 * ring_buffer_write - write data to the buffer without reserving
 * @buffer: The ring buffer to write to.
 * @length: The length of the data being written (excluding the event header)
 * @data: The data to write to the buffer.
 *
 * This is like ring_buffer_lock_reserve and ring_buffer_unlock_commit as
 * one function. If you already have the data to write to the buffer, it
 * may be easier to simply call this function.
 *
 * Note, like ring_buffer_lock_reserve, the length is the length of the data
 * and not the length of the event which would hold the header.
 */
int ring_buffer_write(struct trace_buffer *buffer,
                      unsigned long length,
                      void *data)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        struct ring_buffer_event *event;
        void *body;
        int ret = -EBUSY;
        int cpu;

        preempt_disable_notrace();

        if (atomic_read(&buffer->record_disabled))
                goto out;

        cpu = raw_smp_processor_id();

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                goto out;

        cpu_buffer = buffer->buffers[cpu];

        if (atomic_read(&cpu_buffer->record_disabled))
                goto out;

        if (length > buffer->max_data_size)
                goto out;

        if (unlikely(trace_recursive_lock(cpu_buffer)))
                goto out;

        event = rb_reserve_next_event(buffer, cpu_buffer, length);
        if (!event)
                goto out_unlock;

        body = rb_event_data(event);

        memcpy(body, data, length);

        rb_commit(cpu_buffer);

        rb_wakeups(buffer, cpu_buffer);

        ret = 0;

 out_unlock:
        trace_recursive_unlock(cpu_buffer);

 out:
        preempt_enable_notrace();

        return ret;
}
EXPORT_SYMBOL_GPL(ring_buffer_write);

/*
 * The total entries in the ring buffer is the running counter
 * of entries entered into the ring buffer, minus the sum of
 * the entries read from the ring buffer and the number of
 * entries that were overwritten.
 */
static inline unsigned long
rb_num_of_entries(struct ring_buffer_per_cpu *cpu_buffer)
{
        return local_read(&cpu_buffer->entries) -
                (local_read(&cpu_buffer->overrun) + cpu_buffer->read);
}

static bool rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer)
{
        return !rb_num_of_entries(cpu_buffer);
}

/**
 * ring_buffer_record_disable - stop all writes into the buffer
 * @buffer: The ring buffer to stop writes to.
 *
 * This prevents all writes to the buffer. Any attempt to write
 * to the buffer after this will fail and return NULL.
 *
 * The caller should call synchronize_rcu() after this.
 */
void ring_buffer_record_disable(struct trace_buffer *buffer)
{
        atomic_inc(&buffer->record_disabled);
}
EXPORT_SYMBOL_GPL(ring_buffer_record_disable);

/**
 * ring_buffer_record_enable - enable writes to the buffer
 * @buffer: The ring buffer to enable writes
 *
 * Note, multiple disables will need the same number of enables
 * to truly enable the writing (much like preempt_disable).
 */
void ring_buffer_record_enable(struct trace_buffer *buffer)
{
        atomic_dec(&buffer->record_disabled);
}
EXPORT_SYMBOL_GPL(ring_buffer_record_enable);

/**
 * ring_buffer_record_off - stop all writes into the buffer
 * @buffer: The ring buffer to stop writes to.
 *
 * This prevents all writes to the buffer. Any attempt to write
 * to the buffer after this will fail and return NULL.
 *
 * This is different than ring_buffer_record_disable() as
 * it works like an on/off switch, where as the disable() version
 * must be paired with a enable().
 */
void ring_buffer_record_off(struct trace_buffer *buffer)
{
        unsigned int rd;
        unsigned int new_rd;

        rd = atomic_read(&buffer->record_disabled);
        do {
                new_rd = rd | RB_BUFFER_OFF;
        } while (!atomic_try_cmpxchg(&buffer->record_disabled, &rd, new_rd));
}
EXPORT_SYMBOL_GPL(ring_buffer_record_off);

/**
 * ring_buffer_record_on - restart writes into the buffer
 * @buffer: The ring buffer to start writes to.
 *
 * This enables all writes to the buffer that was disabled by
 * ring_buffer_record_off().
 *
 * This is different than ring_buffer_record_enable() as
 * it works like an on/off switch, where as the enable() version
 * must be paired with a disable().
 */
void ring_buffer_record_on(struct trace_buffer *buffer)
{
        unsigned int rd;
        unsigned int new_rd;

        rd = atomic_read(&buffer->record_disabled);
        do {
                new_rd = rd & ~RB_BUFFER_OFF;
        } while (!atomic_try_cmpxchg(&buffer->record_disabled, &rd, new_rd));
}
EXPORT_SYMBOL_GPL(ring_buffer_record_on);

/**
 * ring_buffer_record_is_on - return true if the ring buffer can write
 * @buffer: The ring buffer to see if write is enabled
 *
 * Returns true if the ring buffer is in a state that it accepts writes.
 */
bool ring_buffer_record_is_on(struct trace_buffer *buffer)
{
        return !atomic_read(&buffer->record_disabled);
}

/**
 * ring_buffer_record_is_set_on - return true if the ring buffer is set writable
 * @buffer: The ring buffer to see if write is set enabled
 *
 * Returns true if the ring buffer is set writable by ring_buffer_record_on().
 * Note that this does NOT mean it is in a writable state.
 *
 * It may return true when the ring buffer has been disabled by
 * ring_buffer_record_disable(), as that is a temporary disabling of
 * the ring buffer.
 */
bool ring_buffer_record_is_set_on(struct trace_buffer *buffer)
{
        return !(atomic_read(&buffer->record_disabled) & RB_BUFFER_OFF);
}

/**
 * ring_buffer_record_disable_cpu - stop all writes into the cpu_buffer
 * @buffer: The ring buffer to stop writes to.
 * @cpu: The CPU buffer to stop
 *
 * This prevents all writes to the buffer. Any attempt to write
 * to the buffer after this will fail and return NULL.
 *
 * The caller should call synchronize_rcu() after this.
 */
void ring_buffer_record_disable_cpu(struct trace_buffer *buffer, int cpu)
{
        struct ring_buffer_per_cpu *cpu_buffer;

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                return;

        cpu_buffer = buffer->buffers[cpu];
        atomic_inc(&cpu_buffer->record_disabled);
}
EXPORT_SYMBOL_GPL(ring_buffer_record_disable_cpu);

/**
 * ring_buffer_record_enable_cpu - enable writes to the buffer
 * @buffer: The ring buffer to enable writes
 * @cpu: The CPU to enable.
 *
 * Note, multiple disables will need the same number of enables
 * to truly enable the writing (much like preempt_disable).
 */
void ring_buffer_record_enable_cpu(struct trace_buffer *buffer, int cpu)
{
        struct ring_buffer_per_cpu *cpu_buffer;

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                return;

        cpu_buffer = buffer->buffers[cpu];
        atomic_dec(&cpu_buffer->record_disabled);
}
EXPORT_SYMBOL_GPL(ring_buffer_record_enable_cpu);

/**
 * ring_buffer_oldest_event_ts - get the oldest event timestamp from the buffer
 * @buffer: The ring buffer
 * @cpu: The per CPU buffer to read from.
 */
u64 ring_buffer_oldest_event_ts(struct trace_buffer *buffer, int cpu)
{
        unsigned long flags;
        struct ring_buffer_per_cpu *cpu_buffer;
        struct buffer_page *bpage;
        u64 ret = 0;

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                return 0;

        cpu_buffer = buffer->buffers[cpu];
        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
        /*
         * if the tail is on reader_page, oldest time stamp is on the reader
         * page
         */
        if (cpu_buffer->tail_page == cpu_buffer->reader_page)
                bpage = cpu_buffer->reader_page;
        else
                bpage = rb_set_head_page(cpu_buffer);
        if (bpage)
                ret = bpage->page->time_stamp;
        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);

        return ret;
}
EXPORT_SYMBOL_GPL(ring_buffer_oldest_event_ts);

/**
 * ring_buffer_bytes_cpu - get the number of bytes unconsumed in a cpu buffer
 * @buffer: The ring buffer
 * @cpu: The per CPU buffer to read from.
 */
unsigned long ring_buffer_bytes_cpu(struct trace_buffer *buffer, int cpu)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        unsigned long ret;

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                return 0;

        cpu_buffer = buffer->buffers[cpu];
        ret = local_read(&cpu_buffer->entries_bytes) - cpu_buffer->read_bytes;

        return ret;
}
EXPORT_SYMBOL_GPL(ring_buffer_bytes_cpu);

/**
 * ring_buffer_entries_cpu - get the number of entries in a cpu buffer
 * @buffer: The ring buffer
 * @cpu: The per CPU buffer to get the entries from.
 */
unsigned long ring_buffer_entries_cpu(struct trace_buffer *buffer, int cpu)
{
        struct ring_buffer_per_cpu *cpu_buffer;

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                return 0;

        cpu_buffer = buffer->buffers[cpu];

        return rb_num_of_entries(cpu_buffer);
}
EXPORT_SYMBOL_GPL(ring_buffer_entries_cpu);

/**
 * ring_buffer_overrun_cpu - get the number of overruns caused by the ring
 * buffer wrapping around (only if RB_FL_OVERWRITE is on).
 * @buffer: The ring buffer
 * @cpu: The per CPU buffer to get the number of overruns from
 */
unsigned long ring_buffer_overrun_cpu(struct trace_buffer *buffer, int cpu)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        unsigned long ret;

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                return 0;

        cpu_buffer = buffer->buffers[cpu];
        ret = local_read(&cpu_buffer->overrun);

        return ret;
}
EXPORT_SYMBOL_GPL(ring_buffer_overrun_cpu);

/**
 * ring_buffer_commit_overrun_cpu - get the number of overruns caused by
 * commits failing due to the buffer wrapping around while there are uncommitted
 * events, such as during an interrupt storm.
 * @buffer: The ring buffer
 * @cpu: The per CPU buffer to get the number of overruns from
 */
unsigned long
ring_buffer_commit_overrun_cpu(struct trace_buffer *buffer, int cpu)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        unsigned long ret;

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                return 0;

        cpu_buffer = buffer->buffers[cpu];
        ret = local_read(&cpu_buffer->commit_overrun);

        return ret;
}
EXPORT_SYMBOL_GPL(ring_buffer_commit_overrun_cpu);

/**
 * ring_buffer_dropped_events_cpu - get the number of dropped events caused by
 * the ring buffer filling up (only if RB_FL_OVERWRITE is off).
 * @buffer: The ring buffer
 * @cpu: The per CPU buffer to get the number of overruns from
 */
unsigned long
ring_buffer_dropped_events_cpu(struct trace_buffer *buffer, int cpu)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        unsigned long ret;

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                return 0;

        cpu_buffer = buffer->buffers[cpu];
        ret = local_read(&cpu_buffer->dropped_events);

        return ret;
}
EXPORT_SYMBOL_GPL(ring_buffer_dropped_events_cpu);

/**
 * ring_buffer_read_events_cpu - get the number of events successfully read
 * @buffer: The ring buffer
 * @cpu: The per CPU buffer to get the number of events read
 */
unsigned long
ring_buffer_read_events_cpu(struct trace_buffer *buffer, int cpu)
{
        struct ring_buffer_per_cpu *cpu_buffer;

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                return 0;

        cpu_buffer = buffer->buffers[cpu];
        return cpu_buffer->read;
}
EXPORT_SYMBOL_GPL(ring_buffer_read_events_cpu);

/**
 * ring_buffer_entries - get the number of entries in a buffer
 * @buffer: The ring buffer
 *
 * Returns the total number of entries in the ring buffer
 * (all CPU entries)
 */
unsigned long ring_buffer_entries(struct trace_buffer *buffer)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        unsigned long entries = 0;
        int cpu;

        /* if you care about this being correct, lock the buffer */
        for_each_buffer_cpu(buffer, cpu) {
                cpu_buffer = buffer->buffers[cpu];
                entries += rb_num_of_entries(cpu_buffer);
        }

        return entries;
}
EXPORT_SYMBOL_GPL(ring_buffer_entries);

/**
 * ring_buffer_overruns - get the number of overruns in buffer
 * @buffer: The ring buffer
 *
 * Returns the total number of overruns in the ring buffer
 * (all CPU entries)
 */
unsigned long ring_buffer_overruns(struct trace_buffer *buffer)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        unsigned long overruns = 0;
        int cpu;

        /* if you care about this being correct, lock the buffer */
        for_each_buffer_cpu(buffer, cpu) {
                cpu_buffer = buffer->buffers[cpu];
                overruns += local_read(&cpu_buffer->overrun);
        }

        return overruns;
}
EXPORT_SYMBOL_GPL(ring_buffer_overruns);

static void rb_iter_reset(struct ring_buffer_iter *iter)
{
        struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;

        /* Iterator usage is expected to have record disabled */
        iter->head_page = cpu_buffer->reader_page;
        iter->head = cpu_buffer->reader_page->read;
        iter->next_event = iter->head;

        iter->cache_reader_page = iter->head_page;
        iter->cache_read = cpu_buffer->read;
        iter->cache_pages_removed = cpu_buffer->pages_removed;

        if (iter->head) {
                iter->read_stamp = cpu_buffer->read_stamp;
                iter->page_stamp = cpu_buffer->reader_page->page->time_stamp;
        } else {
                iter->read_stamp = iter->head_page->page->time_stamp;
                iter->page_stamp = iter->read_stamp;
        }
}

/**
 * ring_buffer_iter_reset - reset an iterator
 * @iter: The iterator to reset
 *
 * Resets the iterator, so that it will start from the beginning
 * again.
 */
void ring_buffer_iter_reset(struct ring_buffer_iter *iter)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        unsigned long flags;

        if (!iter)
                return;

        cpu_buffer = iter->cpu_buffer;

        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
        rb_iter_reset(iter);
        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
}
EXPORT_SYMBOL_GPL(ring_buffer_iter_reset);

/**
 * ring_buffer_iter_empty - check if an iterator has no more to read
 * @iter: The iterator to check
 */
int ring_buffer_iter_empty(struct ring_buffer_iter *iter)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        struct buffer_page *reader;
        struct buffer_page *head_page;
        struct buffer_page *commit_page;
        struct buffer_page *curr_commit_page;
        unsigned commit;
        u64 curr_commit_ts;
        u64 commit_ts;

        cpu_buffer = iter->cpu_buffer;
        reader = cpu_buffer->reader_page;
        head_page = cpu_buffer->head_page;
        commit_page = READ_ONCE(cpu_buffer->commit_page);
        commit_ts = commit_page->page->time_stamp;

        /*
         * When the writer goes across pages, it issues a cmpxchg which
         * is a mb(), which will synchronize with the rmb here.
         * (see rb_tail_page_update())
         */
        smp_rmb();
        commit = rb_page_commit(commit_page);
        /* We want to make sure that the commit page doesn't change */
        smp_rmb();

        /* Make sure commit page didn't change */
        curr_commit_page = READ_ONCE(cpu_buffer->commit_page);
        curr_commit_ts = READ_ONCE(curr_commit_page->page->time_stamp);

        /* If the commit page changed, then there's more data */
        if (curr_commit_page != commit_page ||
            curr_commit_ts != commit_ts)
                return 0;

        /* Still racy, as it may return a false positive, but that's OK */
        return ((iter->head_page == commit_page && iter->head >= commit) ||
                (iter->head_page == reader && commit_page == head_page &&
                 head_page->read == commit &&
                 iter->head == rb_page_size(cpu_buffer->reader_page)));
}
EXPORT_SYMBOL_GPL(ring_buffer_iter_empty);

static void
rb_update_read_stamp(struct ring_buffer_per_cpu *cpu_buffer,
                     struct ring_buffer_event *event)
{
        u64 delta;

        switch (event->type_len) {
        case RINGBUF_TYPE_PADDING:
                return;

        case RINGBUF_TYPE_TIME_EXTEND:
                delta = rb_event_time_stamp(event);
                cpu_buffer->read_stamp += delta;
                return;

        case RINGBUF_TYPE_TIME_STAMP:
                delta = rb_event_time_stamp(event);
                delta = rb_fix_abs_ts(delta, cpu_buffer->read_stamp);
                cpu_buffer->read_stamp = delta;
                return;

        case RINGBUF_TYPE_DATA:
                cpu_buffer->read_stamp += event->time_delta;
                return;

        default:
                RB_WARN_ON(cpu_buffer, 1);
        }
}

static void
rb_update_iter_read_stamp(struct ring_buffer_iter *iter,
                          struct ring_buffer_event *event)
{
        u64 delta;

        switch (event->type_len) {
        case RINGBUF_TYPE_PADDING:
                return;

        case RINGBUF_TYPE_TIME_EXTEND:
                delta = rb_event_time_stamp(event);
                iter->read_stamp += delta;
                return;

        case RINGBUF_TYPE_TIME_STAMP:
                delta = rb_event_time_stamp(event);
                delta = rb_fix_abs_ts(delta, iter->read_stamp);
                iter->read_stamp = delta;
                return;

        case RINGBUF_TYPE_DATA:
                iter->read_stamp += event->time_delta;
                return;

        default:
                RB_WARN_ON(iter->cpu_buffer, 1);
        }
}

static struct buffer_page *
rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
{
        struct buffer_page *reader = NULL;
        unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size);
        unsigned long overwrite;
        unsigned long flags;
        int nr_loops = 0;
        bool ret;

        local_irq_save(flags);
        arch_spin_lock(&cpu_buffer->lock);

 again:
        /*
         * This should normally only loop twice. But because the
         * start of the reader inserts an empty page, it causes
         * a case where we will loop three times. There should be no
         * reason to loop four times (that I know of).
         */
        if (RB_WARN_ON(cpu_buffer, ++nr_loops > 3)) {
                reader = NULL;
                goto out;
        }

        reader = cpu_buffer->reader_page;

        /* If there's more to read, return this page */
        if (cpu_buffer->reader_page->read < rb_page_size(reader))
                goto out;

        /* Never should we have an index greater than the size */
        if (RB_WARN_ON(cpu_buffer,
                       cpu_buffer->reader_page->read > rb_page_size(reader)))
                goto out;

        /* check if we caught up to the tail */
        reader = NULL;
        if (cpu_buffer->commit_page == cpu_buffer->reader_page)
                goto out;

        /* Don't bother swapping if the ring buffer is empty */
        if (rb_num_of_entries(cpu_buffer) == 0)
                goto out;

        /*
         * Reset the reader page to size zero.
         */
        local_set(&cpu_buffer->reader_page->write, 0);
        local_set(&cpu_buffer->reader_page->entries, 0);
        local_set(&cpu_buffer->reader_page->page->commit, 0);
        cpu_buffer->reader_page->real_end = 0;

 spin:
        /*
         * Splice the empty reader page into the list around the head.
         */
        reader = rb_set_head_page(cpu_buffer);
        if (!reader)
                goto out;
        cpu_buffer->reader_page->list.next = rb_list_head(reader->list.next);
        cpu_buffer->reader_page->list.prev = reader->list.prev;

        /*
         * cpu_buffer->pages just needs to point to the buffer, it
         *  has no specific buffer page to point to. Lets move it out
         *  of our way so we don't accidentally swap it.
         */
        cpu_buffer->pages = reader->list.prev;

        /* The reader page will be pointing to the new head */
        rb_set_list_to_head(&cpu_buffer->reader_page->list);

        /*
         * We want to make sure we read the overruns after we set up our
         * pointers to the next object. The writer side does a
         * cmpxchg to cross pages which acts as the mb on the writer
         * side. Note, the reader will constantly fail the swap
         * while the writer is updating the pointers, so this
         * guarantees that the overwrite recorded here is the one we
         * want to compare with the last_overrun.
         */
        smp_mb();
        overwrite = local_read(&(cpu_buffer->overrun));

        /*
         * Here's the tricky part.
         *
         * We need to move the pointer past the header page.
         * But we can only do that if a writer is not currently
         * moving it. The page before the header page has the
         * flag bit '1' set if it is pointing to the page we want.
         * but if the writer is in the process of moving it
         * then it will be '2' or already moved '0'.
         */

        ret = rb_head_page_replace(reader, cpu_buffer->reader_page);

        /*
         * If we did not convert it, then we must try again.
         */
        if (!ret)
                goto spin;

        if (cpu_buffer->ring_meta)
                rb_update_meta_reader(cpu_buffer, reader);

        /*
         * Yay! We succeeded in replacing the page.
         *
         * Now make the new head point back to the reader page.
         */
        rb_list_head(reader->list.next)->prev = &cpu_buffer->reader_page->list;
        rb_inc_page(&cpu_buffer->head_page);

        cpu_buffer->cnt++;
        local_inc(&cpu_buffer->pages_read);

        /* Finally update the reader page to the new head */
        cpu_buffer->reader_page = reader;
        cpu_buffer->reader_page->read = 0;

        if (overwrite != cpu_buffer->last_overrun) {
                cpu_buffer->lost_events = overwrite - cpu_buffer->last_overrun;
                cpu_buffer->last_overrun = overwrite;
        }

        goto again;

 out:
        /* Update the read_stamp on the first event */
        if (reader && reader->read == 0)
                cpu_buffer->read_stamp = reader->page->time_stamp;

        arch_spin_unlock(&cpu_buffer->lock);
        local_irq_restore(flags);

        /*
         * The writer has preempt disable, wait for it. But not forever
         * Although, 1 second is pretty much "forever"
         */
#define USECS_WAIT        1000000
        for (nr_loops = 0; nr_loops < USECS_WAIT; nr_loops++) {
                /* If the write is past the end of page, a writer is still updating it */
                if (likely(!reader || rb_page_write(reader) <= bsize))
                        break;

                udelay(1);

                /* Get the latest version of the reader write value */
                smp_rmb();
        }

        /* The writer is not moving forward? Something is wrong */
        if (RB_WARN_ON(cpu_buffer, nr_loops == USECS_WAIT))
                reader = NULL;

        /*
         * Make sure we see any padding after the write update
         * (see rb_reset_tail()).
         *
         * In addition, a writer may be writing on the reader page
         * if the page has not been fully filled, so the read barrier
         * is also needed to make sure we see the content of what is
         * committed by the writer (see rb_set_commit_to_write()).
         */
        smp_rmb();


        return reader;
}

static void rb_advance_reader(struct ring_buffer_per_cpu *cpu_buffer)
{
        struct ring_buffer_event *event;
        struct buffer_page *reader;
        unsigned length;

        reader = rb_get_reader_page(cpu_buffer);

        /* This function should not be called when buffer is empty */
        if (RB_WARN_ON(cpu_buffer, !reader))
                return;

        event = rb_reader_event(cpu_buffer);

        if (event->type_len <= RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
                cpu_buffer->read++;

        rb_update_read_stamp(cpu_buffer, event);

        length = rb_event_length(event);
        cpu_buffer->reader_page->read += length;
        cpu_buffer->read_bytes += length;
}

static void rb_advance_iter(struct ring_buffer_iter *iter)
{
        struct ring_buffer_per_cpu *cpu_buffer;

        cpu_buffer = iter->cpu_buffer;

        /* If head == next_event then we need to jump to the next event */
        if (iter->head == iter->next_event) {
                /* If the event gets overwritten again, there's nothing to do */
                if (rb_iter_head_event(iter) == NULL)
                        return;
        }

        iter->head = iter->next_event;

        /*
         * Check if we are at the end of the buffer.
         */
        if (iter->next_event >= rb_page_size(iter->head_page)) {
                /* discarded commits can make the page empty */
                if (iter->head_page == cpu_buffer->commit_page)
                        return;
                rb_inc_iter(iter);
                return;
        }

        rb_update_iter_read_stamp(iter, iter->event);
}

static int rb_lost_events(struct ring_buffer_per_cpu *cpu_buffer)
{
        return cpu_buffer->lost_events;
}

static struct ring_buffer_event *
rb_buffer_peek(struct ring_buffer_per_cpu *cpu_buffer, u64 *ts,
               unsigned long *lost_events)
{
        struct ring_buffer_event *event;
        struct buffer_page *reader;
        int nr_loops = 0;

        if (ts)
                *ts = 0;
 again:
        /*
         * We repeat when a time extend is encountered.
         * Since the time extend is always attached to a data event,
         * we should never loop more than once.
         * (We never hit the following condition more than twice).
         */
        if (RB_WARN_ON(cpu_buffer, ++nr_loops > 2))
                return NULL;

        reader = rb_get_reader_page(cpu_buffer);
        if (!reader)
                return NULL;

        event = rb_reader_event(cpu_buffer);

        switch (event->type_len) {
        case RINGBUF_TYPE_PADDING:
                if (rb_null_event(event))
                        RB_WARN_ON(cpu_buffer, 1);
                /*
                 * Because the writer could be discarding every
                 * event it creates (which would probably be bad)
                 * if we were to go back to "again" then we may never
                 * catch up, and will trigger the warn on, or lock
                 * the box. Return the padding, and we will release
                 * the current locks, and try again.
                 */
                return event;

        case RINGBUF_TYPE_TIME_EXTEND:
                /* Internal data, OK to advance */
                rb_advance_reader(cpu_buffer);
                goto again;

        case RINGBUF_TYPE_TIME_STAMP:
                if (ts) {
                        *ts = rb_event_time_stamp(event);
                        *ts = rb_fix_abs_ts(*ts, reader->page->time_stamp);
                        ring_buffer_normalize_time_stamp(cpu_buffer->buffer,
                                                         cpu_buffer->cpu, ts);
                }
                /* Internal data, OK to advance */
                rb_advance_reader(cpu_buffer);
                goto again;

        case RINGBUF_TYPE_DATA:
                if (ts && !(*ts)) {
                        *ts = cpu_buffer->read_stamp + event->time_delta;
                        ring_buffer_normalize_time_stamp(cpu_buffer->buffer,
                                                         cpu_buffer->cpu, ts);
                }
                if (lost_events)
                        *lost_events = rb_lost_events(cpu_buffer);
                return event;

        default:
                RB_WARN_ON(cpu_buffer, 1);
        }

        return NULL;
}
EXPORT_SYMBOL_GPL(ring_buffer_peek);

static struct ring_buffer_event *
rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
{
        struct trace_buffer *buffer;
        struct ring_buffer_per_cpu *cpu_buffer;
        struct ring_buffer_event *event;
        int nr_loops = 0;

        if (ts)
                *ts = 0;

        cpu_buffer = iter->cpu_buffer;
        buffer = cpu_buffer->buffer;

        /*
         * Check if someone performed a consuming read to the buffer
         * or removed some pages from the buffer. In these cases,
         * iterator was invalidated and we need to reset it.
         */
        if (unlikely(iter->cache_read != cpu_buffer->read ||
                     iter->cache_reader_page != cpu_buffer->reader_page ||
                     iter->cache_pages_removed != cpu_buffer->pages_removed))
                rb_iter_reset(iter);

 again:
        if (ring_buffer_iter_empty(iter))
                return NULL;

        /*
         * As the writer can mess with what the iterator is trying
         * to read, just give up if we fail to get an event after
         * three tries. The iterator is not as reliable when reading
         * the ring buffer with an active write as the consumer is.
         * Do not warn if the three failures is reached.
         */
        if (++nr_loops > 3)
                return NULL;

        if (rb_per_cpu_empty(cpu_buffer))
                return NULL;

        if (iter->head >= rb_page_size(iter->head_page)) {
                rb_inc_iter(iter);
                goto again;
        }

        event = rb_iter_head_event(iter);
        if (!event)
                goto again;

        switch (event->type_len) {
        case RINGBUF_TYPE_PADDING:
                if (rb_null_event(event)) {
                        rb_inc_iter(iter);
                        goto again;
                }
                rb_advance_iter(iter);
                return event;

        case RINGBUF_TYPE_TIME_EXTEND:
                /* Internal data, OK to advance */
                rb_advance_iter(iter);
                goto again;

        case RINGBUF_TYPE_TIME_STAMP:
                if (ts) {
                        *ts = rb_event_time_stamp(event);
                        *ts = rb_fix_abs_ts(*ts, iter->head_page->page->time_stamp);
                        ring_buffer_normalize_time_stamp(cpu_buffer->buffer,
                                                         cpu_buffer->cpu, ts);
                }
                /* Internal data, OK to advance */
                rb_advance_iter(iter);
                goto again;

        case RINGBUF_TYPE_DATA:
                if (ts && !(*ts)) {
                        *ts = iter->read_stamp + event->time_delta;
                        ring_buffer_normalize_time_stamp(buffer,
                                                         cpu_buffer->cpu, ts);
                }
                return event;

        default:
                RB_WARN_ON(cpu_buffer, 1);
        }

        return NULL;
}
EXPORT_SYMBOL_GPL(ring_buffer_iter_peek);

static inline bool rb_reader_lock(struct ring_buffer_per_cpu *cpu_buffer)
{
        if (likely(!in_nmi())) {
                raw_spin_lock(&cpu_buffer->reader_lock);
                return true;
        }

        /*
         * If an NMI die dumps out the content of the ring buffer
         * trylock must be used to prevent a deadlock if the NMI
         * preempted a task that holds the ring buffer locks. If
         * we get the lock then all is fine, if not, then continue
         * to do the read, but this can corrupt the ring buffer,
         * so it must be permanently disabled from future writes.
         * Reading from NMI is a oneshot deal.
         */
        if (raw_spin_trylock(&cpu_buffer->reader_lock))
                return true;

        /* Continue without locking, but disable the ring buffer */
        atomic_inc(&cpu_buffer->record_disabled);
        return false;
}

static inline void
rb_reader_unlock(struct ring_buffer_per_cpu *cpu_buffer, bool locked)
{
        if (likely(locked))
                raw_spin_unlock(&cpu_buffer->reader_lock);
}

/**
 * ring_buffer_peek - peek at the next event to be read
 * @buffer: The ring buffer to read
 * @cpu: The cpu to peak at
 * @ts: The timestamp counter of this event.
 * @lost_events: a variable to store if events were lost (may be NULL)
 *
 * This will return the event that will be read next, but does
 * not consume the data.
 */
struct ring_buffer_event *
ring_buffer_peek(struct trace_buffer *buffer, int cpu, u64 *ts,
                 unsigned long *lost_events)
{
        struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
        struct ring_buffer_event *event;
        unsigned long flags;
        bool dolock;

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                return NULL;

 again:
        local_irq_save(flags);
        dolock = rb_reader_lock(cpu_buffer);
        event = rb_buffer_peek(cpu_buffer, ts, lost_events);
        if (event && event->type_len == RINGBUF_TYPE_PADDING)
                rb_advance_reader(cpu_buffer);
        rb_reader_unlock(cpu_buffer, dolock);
        local_irq_restore(flags);

        if (event && event->type_len == RINGBUF_TYPE_PADDING)
                goto again;

        return event;
}

/** ring_buffer_iter_dropped - report if there are dropped events
 * @iter: The ring buffer iterator
 *
 * Returns true if there was dropped events since the last peek.
 */
bool ring_buffer_iter_dropped(struct ring_buffer_iter *iter)
{
        bool ret = iter->missed_events != 0;

        iter->missed_events = 0;
        return ret;
}
EXPORT_SYMBOL_GPL(ring_buffer_iter_dropped);

/**
 * ring_buffer_iter_peek - peek at the next event to be read
 * @iter: The ring buffer iterator
 * @ts: The timestamp counter of this event.
 *
 * This will return the event that will be read next, but does
 * not increment the iterator.
 */
struct ring_buffer_event *
ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
{
        struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
        struct ring_buffer_event *event;
        unsigned long flags;

 again:
        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
        event = rb_iter_peek(iter, ts);
        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);

        if (event && event->type_len == RINGBUF_TYPE_PADDING)
                goto again;

        return event;
}

/**
 * ring_buffer_consume - return an event and consume it
 * @buffer: The ring buffer to get the next event from
 * @cpu: the cpu to read the buffer from
 * @ts: a variable to store the timestamp (may be NULL)
 * @lost_events: a variable to store if events were lost (may be NULL)
 *
 * Returns the next event in the ring buffer, and that event is consumed.
 * Meaning, that sequential reads will keep returning a different event,
 * and eventually empty the ring buffer if the producer is slower.
 */
struct ring_buffer_event *
ring_buffer_consume(struct trace_buffer *buffer, int cpu, u64 *ts,
                    unsigned long *lost_events)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        struct ring_buffer_event *event = NULL;
        unsigned long flags;
        bool dolock;

 again:
        /* might be called in atomic */
        preempt_disable();

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                goto out;

        cpu_buffer = buffer->buffers[cpu];
        local_irq_save(flags);
        dolock = rb_reader_lock(cpu_buffer);

        event = rb_buffer_peek(cpu_buffer, ts, lost_events);
        if (event) {
                cpu_buffer->lost_events = 0;
                rb_advance_reader(cpu_buffer);
        }

        rb_reader_unlock(cpu_buffer, dolock);
        local_irq_restore(flags);

 out:
        preempt_enable();

        if (event && event->type_len == RINGBUF_TYPE_PADDING)
                goto again;

        return event;
}
EXPORT_SYMBOL_GPL(ring_buffer_consume);

/**
 * ring_buffer_read_prepare - Prepare for a non consuming read of the buffer
 * @buffer: The ring buffer to read from
 * @cpu: The cpu buffer to iterate over
 * @flags: gfp flags to use for memory allocation
 *
 * This performs the initial preparations necessary to iterate
 * through the buffer.  Memory is allocated, buffer resizing
 * is disabled, and the iterator pointer is returned to the caller.
 *
 * After a sequence of ring_buffer_read_prepare calls, the user is
 * expected to make at least one call to ring_buffer_read_prepare_sync.
 * Afterwards, ring_buffer_read_start is invoked to get things going
 * for real.
 *
 * This overall must be paired with ring_buffer_read_finish.
 */
struct ring_buffer_iter *
ring_buffer_read_prepare(struct trace_buffer *buffer, int cpu, gfp_t flags)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        struct ring_buffer_iter *iter;

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                return NULL;

        iter = kzalloc(sizeof(*iter), flags);
        if (!iter)
                return NULL;

        /* Holds the entire event: data and meta data */
        iter->event_size = buffer->subbuf_size;
        iter->event = kmalloc(iter->event_size, flags);
        if (!iter->event) {
                kfree(iter);
                return NULL;
        }

        cpu_buffer = buffer->buffers[cpu];

        iter->cpu_buffer = cpu_buffer;

        atomic_inc(&cpu_buffer->resize_disabled);

        return iter;
}
EXPORT_SYMBOL_GPL(ring_buffer_read_prepare);

/**
 * ring_buffer_read_prepare_sync - Synchronize a set of prepare calls
 *
 * All previously invoked ring_buffer_read_prepare calls to prepare
 * iterators will be synchronized.  Afterwards, read_buffer_read_start
 * calls on those iterators are allowed.
 */
void
ring_buffer_read_prepare_sync(void)
{
        synchronize_rcu();
}
EXPORT_SYMBOL_GPL(ring_buffer_read_prepare_sync);

/**
 * ring_buffer_read_start - start a non consuming read of the buffer
 * @iter: The iterator returned by ring_buffer_read_prepare
 *
 * This finalizes the startup of an iteration through the buffer.
 * The iterator comes from a call to ring_buffer_read_prepare and
 * an intervening ring_buffer_read_prepare_sync must have been
 * performed.
 *
 * Must be paired with ring_buffer_read_finish.
 */
void
ring_buffer_read_start(struct ring_buffer_iter *iter)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        unsigned long flags;

        if (!iter)
                return;

        cpu_buffer = iter->cpu_buffer;

        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
        arch_spin_lock(&cpu_buffer->lock);
        rb_iter_reset(iter);
        arch_spin_unlock(&cpu_buffer->lock);
        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
}
EXPORT_SYMBOL_GPL(ring_buffer_read_start);

/**
 * ring_buffer_read_finish - finish reading the iterator of the buffer
 * @iter: The iterator retrieved by ring_buffer_start
 *
 * This re-enables resizing of the buffer, and frees the iterator.
 */
void
ring_buffer_read_finish(struct ring_buffer_iter *iter)
{
        struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;

        /* Use this opportunity to check the integrity of the ring buffer. */
        rb_check_pages(cpu_buffer);

        atomic_dec(&cpu_buffer->resize_disabled);
        kfree(iter->event);
        kfree(iter);
}
EXPORT_SYMBOL_GPL(ring_buffer_read_finish);

/**
 * ring_buffer_iter_advance - advance the iterator to the next location
 * @iter: The ring buffer iterator
 *
 * Move the location of the iterator such that the next read will
 * be the next location of the iterator.
 */
void ring_buffer_iter_advance(struct ring_buffer_iter *iter)
{
        struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
        unsigned long flags;

        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);

        rb_advance_iter(iter);

        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
}
EXPORT_SYMBOL_GPL(ring_buffer_iter_advance);

/**
 * ring_buffer_size - return the size of the ring buffer (in bytes)
 * @buffer: The ring buffer.
 * @cpu: The CPU to get ring buffer size from.
 */
unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu)
{
        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                return 0;

        return buffer->subbuf_size * buffer->buffers[cpu]->nr_pages;
}
EXPORT_SYMBOL_GPL(ring_buffer_size);

/**
 * ring_buffer_max_event_size - return the max data size of an event
 * @buffer: The ring buffer.
 *
 * Returns the maximum size an event can be.
 */
unsigned long ring_buffer_max_event_size(struct trace_buffer *buffer)
{
        /* If abs timestamp is requested, events have a timestamp too */
        if (ring_buffer_time_stamp_abs(buffer))
                return buffer->max_data_size - RB_LEN_TIME_EXTEND;
        return buffer->max_data_size;
}
EXPORT_SYMBOL_GPL(ring_buffer_max_event_size);

static void rb_clear_buffer_page(struct buffer_page *page)
{
        local_set(&page->write, 0);
        local_set(&page->entries, 0);
        rb_init_page(page->page);
        page->read = 0;
}

static void rb_update_meta_page(struct ring_buffer_per_cpu *cpu_buffer)
{
        struct trace_buffer_meta *meta = cpu_buffer->meta_page;

        if (!meta)
                return;

        meta->reader.read = cpu_buffer->reader_page->read;
        meta->reader.id = cpu_buffer->reader_page->id;
        meta->reader.lost_events = cpu_buffer->lost_events;

        meta->entries = local_read(&cpu_buffer->entries);
        meta->overrun = local_read(&cpu_buffer->overrun);
        meta->read = cpu_buffer->read;

        /* Some archs do not have data cache coherency between kernel and user-space */
        flush_kernel_vmap_range(cpu_buffer->meta_page, PAGE_SIZE);
}

static void
rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
{
        struct buffer_page *page;

        rb_head_page_deactivate(cpu_buffer);

        cpu_buffer->head_page
                = list_entry(cpu_buffer->pages, struct buffer_page, list);
        rb_clear_buffer_page(cpu_buffer->head_page);
        list_for_each_entry(page, cpu_buffer->pages, list) {
                rb_clear_buffer_page(page);
        }

        cpu_buffer->tail_page = cpu_buffer->head_page;
        cpu_buffer->commit_page = cpu_buffer->head_page;

        INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
        INIT_LIST_HEAD(&cpu_buffer->new_pages);
        rb_clear_buffer_page(cpu_buffer->reader_page);

        local_set(&cpu_buffer->entries_bytes, 0);
        local_set(&cpu_buffer->overrun, 0);
        local_set(&cpu_buffer->commit_overrun, 0);
        local_set(&cpu_buffer->dropped_events, 0);
        local_set(&cpu_buffer->entries, 0);
        local_set(&cpu_buffer->committing, 0);
        local_set(&cpu_buffer->commits, 0);
        local_set(&cpu_buffer->pages_touched, 0);
        local_set(&cpu_buffer->pages_lost, 0);
        local_set(&cpu_buffer->pages_read, 0);
        cpu_buffer->last_pages_touch = 0;
        cpu_buffer->shortest_full = 0;
        cpu_buffer->read = 0;
        cpu_buffer->read_bytes = 0;

        rb_time_set(&cpu_buffer->write_stamp, 0);
        rb_time_set(&cpu_buffer->before_stamp, 0);

        memset(cpu_buffer->event_stamp, 0, sizeof(cpu_buffer->event_stamp));

        cpu_buffer->lost_events = 0;
        cpu_buffer->last_overrun = 0;

        rb_head_page_activate(cpu_buffer);
        cpu_buffer->pages_removed = 0;

        if (cpu_buffer->mapped) {
                rb_update_meta_page(cpu_buffer);
                if (cpu_buffer->ring_meta) {
                        struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta;
                        meta->commit_buffer = meta->head_buffer;
                }
        }
}

/* Must have disabled the cpu buffer then done a synchronize_rcu */
static void reset_disabled_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer)
{
        unsigned long flags;

        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);

        if (RB_WARN_ON(cpu_buffer, local_read(&cpu_buffer->committing)))
                goto out;

        arch_spin_lock(&cpu_buffer->lock);

        rb_reset_cpu(cpu_buffer);

        arch_spin_unlock(&cpu_buffer->lock);

 out:
        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
}

/**
 * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
 * @buffer: The ring buffer to reset a per cpu buffer of
 * @cpu: The CPU buffer to be reset
 */
void ring_buffer_reset_cpu(struct trace_buffer *buffer, int cpu)
{
        struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                return;

        /* prevent another thread from changing buffer sizes */
        mutex_lock(&buffer->mutex);

        atomic_inc(&cpu_buffer->resize_disabled);
        atomic_inc(&cpu_buffer->record_disabled);

        /* Make sure all commits have finished */
        synchronize_rcu();

        reset_disabled_cpu_buffer(cpu_buffer);

        atomic_dec(&cpu_buffer->record_disabled);
        atomic_dec(&cpu_buffer->resize_disabled);

        mutex_unlock(&buffer->mutex);
}
EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu);

/* Flag to ensure proper resetting of atomic variables */
#define RESET_BIT        (1 << 30)

/**
 * ring_buffer_reset_online_cpus - reset a ring buffer per CPU buffer
 * @buffer: The ring buffer to reset a per cpu buffer of
 */
void ring_buffer_reset_online_cpus(struct trace_buffer *buffer)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        int cpu;

        /* prevent another thread from changing buffer sizes */
        mutex_lock(&buffer->mutex);

        for_each_online_buffer_cpu(buffer, cpu) {
                cpu_buffer = buffer->buffers[cpu];

                atomic_add(RESET_BIT, &cpu_buffer->resize_disabled);
                atomic_inc(&cpu_buffer->record_disabled);
        }

        /* Make sure all commits have finished */
        synchronize_rcu();

        for_each_buffer_cpu(buffer, cpu) {
                cpu_buffer = buffer->buffers[cpu];

                /*
                 * If a CPU came online during the synchronize_rcu(), then
                 * ignore it.
                 */
                if (!(atomic_read(&cpu_buffer->resize_disabled) & RESET_BIT))
                        continue;

                reset_disabled_cpu_buffer(cpu_buffer);

                atomic_dec(&cpu_buffer->record_disabled);
                atomic_sub(RESET_BIT, &cpu_buffer->resize_disabled);
        }

        mutex_unlock(&buffer->mutex);
}

/**
 * ring_buffer_reset - reset a ring buffer
 * @buffer: The ring buffer to reset all cpu buffers
 */
void ring_buffer_reset(struct trace_buffer *buffer)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        int cpu;

        /* prevent another thread from changing buffer sizes */
        mutex_lock(&buffer->mutex);

        for_each_buffer_cpu(buffer, cpu) {
                cpu_buffer = buffer->buffers[cpu];

                atomic_inc(&cpu_buffer->resize_disabled);
                atomic_inc(&cpu_buffer->record_disabled);
        }

        /* Make sure all commits have finished */
        synchronize_rcu();

        for_each_buffer_cpu(buffer, cpu) {
                cpu_buffer = buffer->buffers[cpu];

                reset_disabled_cpu_buffer(cpu_buffer);

                atomic_dec(&cpu_buffer->record_disabled);
                atomic_dec(&cpu_buffer->resize_disabled);
        }

        mutex_unlock(&buffer->mutex);
}
EXPORT_SYMBOL_GPL(ring_buffer_reset);

/**
 * ring_buffer_empty - is the ring buffer empty?
 * @buffer: The ring buffer to test
 */
bool ring_buffer_empty(struct trace_buffer *buffer)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        unsigned long flags;
        bool dolock;
        bool ret;
        int cpu;

        /* yes this is racy, but if you don't like the race, lock the buffer */
        for_each_buffer_cpu(buffer, cpu) {
                cpu_buffer = buffer->buffers[cpu];
                local_irq_save(flags);
                dolock = rb_reader_lock(cpu_buffer);
                ret = rb_per_cpu_empty(cpu_buffer);
                rb_reader_unlock(cpu_buffer, dolock);
                local_irq_restore(flags);

                if (!ret)
                        return false;
        }

        return true;
}
EXPORT_SYMBOL_GPL(ring_buffer_empty);

/**
 * ring_buffer_empty_cpu - is a cpu buffer of a ring buffer empty?
 * @buffer: The ring buffer
 * @cpu: The CPU buffer to test
 */
bool ring_buffer_empty_cpu(struct trace_buffer *buffer, int cpu)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        unsigned long flags;
        bool dolock;
        bool ret;

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                return true;

        cpu_buffer = buffer->buffers[cpu];
        local_irq_save(flags);
        dolock = rb_reader_lock(cpu_buffer);
        ret = rb_per_cpu_empty(cpu_buffer);
        rb_reader_unlock(cpu_buffer, dolock);
        local_irq_restore(flags);

        return ret;
}
EXPORT_SYMBOL_GPL(ring_buffer_empty_cpu);

#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
/**
 * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
 * @buffer_a: One buffer to swap with
 * @buffer_b: The other buffer to swap with
 * @cpu: the CPU of the buffers to swap
 *
 * This function is useful for tracers that want to take a "snapshot"
 * of a CPU buffer and has another back up buffer lying around.
 * it is expected that the tracer handles the cpu buffer not being
 * used at the moment.
 */
int ring_buffer_swap_cpu(struct trace_buffer *buffer_a,
                         struct trace_buffer *buffer_b, int cpu)
{
        struct ring_buffer_per_cpu *cpu_buffer_a;
        struct ring_buffer_per_cpu *cpu_buffer_b;
        int ret = -EINVAL;

        if (!cpumask_test_cpu(cpu, buffer_a->cpumask) ||
            !cpumask_test_cpu(cpu, buffer_b->cpumask))
                goto out;

        cpu_buffer_a = buffer_a->buffers[cpu];
        cpu_buffer_b = buffer_b->buffers[cpu];

        /* It's up to the callers to not try to swap mapped buffers */
        if (WARN_ON_ONCE(cpu_buffer_a->mapped || cpu_buffer_b->mapped)) {
                ret = -EBUSY;
                goto out;
        }

        /* At least make sure the two buffers are somewhat the same */
        if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages)
                goto out;

        if (buffer_a->subbuf_order != buffer_b->subbuf_order)
                goto out;

        ret = -EAGAIN;

        if (atomic_read(&buffer_a->record_disabled))
                goto out;

        if (atomic_read(&buffer_b->record_disabled))
                goto out;

        if (atomic_read(&cpu_buffer_a->record_disabled))
                goto out;

        if (atomic_read(&cpu_buffer_b->record_disabled))
                goto out;

        /*
         * We can't do a synchronize_rcu here because this
         * function can be called in atomic context.
         * Normally this will be called from the same CPU as cpu.
         * If not it's up to the caller to protect this.
         */
        atomic_inc(&cpu_buffer_a->record_disabled);
        atomic_inc(&cpu_buffer_b->record_disabled);

        ret = -EBUSY;
        if (local_read(&cpu_buffer_a->committing))
                goto out_dec;
        if (local_read(&cpu_buffer_b->committing))
                goto out_dec;

        /*
         * When resize is in progress, we cannot swap it because
         * it will mess the state of the cpu buffer.
         */
        if (atomic_read(&buffer_a->resizing))
                goto out_dec;
        if (atomic_read(&buffer_b->resizing))
                goto out_dec;

        buffer_a->buffers[cpu] = cpu_buffer_b;
        buffer_b->buffers[cpu] = cpu_buffer_a;

        cpu_buffer_b->buffer = buffer_a;
        cpu_buffer_a->buffer = buffer_b;

        ret = 0;

out_dec:
        atomic_dec(&cpu_buffer_a->record_disabled);
        atomic_dec(&cpu_buffer_b->record_disabled);
out:
        return ret;
}
EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu);
#endif /* CONFIG_RING_BUFFER_ALLOW_SWAP */

/**
 * ring_buffer_alloc_read_page - allocate a page to read from buffer
 * @buffer: the buffer to allocate for.
 * @cpu: the cpu buffer to allocate.
 *
 * This function is used in conjunction with ring_buffer_read_page.
 * When reading a full page from the ring buffer, these functions
 * can be used to speed up the process. The calling function should
 * allocate a few pages first with this function. Then when it
 * needs to get pages from the ring buffer, it passes the result
 * of this function into ring_buffer_read_page, which will swap
 * the page that was allocated, with the read page of the buffer.
 *
 * Returns:
 *  The page allocated, or ERR_PTR
 */
struct buffer_data_read_page *
ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        struct buffer_data_read_page *bpage = NULL;
        unsigned long flags;
        struct page *page;

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                return ERR_PTR(-ENODEV);

        bpage = kzalloc(sizeof(*bpage), GFP_KERNEL);
        if (!bpage)
                return ERR_PTR(-ENOMEM);

        bpage->order = buffer->subbuf_order;
        cpu_buffer = buffer->buffers[cpu];
        local_irq_save(flags);
        arch_spin_lock(&cpu_buffer->lock);

        if (cpu_buffer->free_page) {
                bpage->data = cpu_buffer->free_page;
                cpu_buffer->free_page = NULL;
        }

        arch_spin_unlock(&cpu_buffer->lock);
        local_irq_restore(flags);

        if (bpage->data)
                goto out;

        page = alloc_pages_node(cpu_to_node(cpu),
                                GFP_KERNEL | __GFP_NORETRY | __GFP_COMP | __GFP_ZERO,
                                cpu_buffer->buffer->subbuf_order);
        if (!page) {
                kfree(bpage);
                return ERR_PTR(-ENOMEM);
        }

        bpage->data = page_address(page);

 out:
        rb_init_page(bpage->data);

        return bpage;
}
EXPORT_SYMBOL_GPL(ring_buffer_alloc_read_page);

/**
 * ring_buffer_free_read_page - free an allocated read page
 * @buffer: the buffer the page was allocate for
 * @cpu: the cpu buffer the page came from
 * @data_page: the page to free
 *
 * Free a page allocated from ring_buffer_alloc_read_page.
 */
void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu,
                                struct buffer_data_read_page *data_page)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        struct buffer_data_page *bpage = data_page->data;
        struct page *page = virt_to_page(bpage);
        unsigned long flags;

        if (!buffer || !buffer->buffers || !buffer->buffers[cpu])
                return;

        cpu_buffer = buffer->buffers[cpu];

        /*
         * If the page is still in use someplace else, or order of the page
         * is different from the subbuffer order of the buffer -
         * we can't reuse it
         */
        if (page_ref_count(page) > 1 || data_page->order != buffer->subbuf_order)
                goto out;

        local_irq_save(flags);
        arch_spin_lock(&cpu_buffer->lock);

        if (!cpu_buffer->free_page) {
                cpu_buffer->free_page = bpage;
                bpage = NULL;
        }

        arch_spin_unlock(&cpu_buffer->lock);
        local_irq_restore(flags);

 out:
        free_pages((unsigned long)bpage, data_page->order);
        kfree(data_page);
}
EXPORT_SYMBOL_GPL(ring_buffer_free_read_page);

/**
 * ring_buffer_read_page - extract a page from the ring buffer
 * @buffer: buffer to extract from
 * @data_page: the page to use allocated from ring_buffer_alloc_read_page
 * @len: amount to extract
 * @cpu: the cpu of the buffer to extract
 * @full: should the extraction only happen when the page is full.
 *
 * This function will pull out a page from the ring buffer and consume it.
 * @data_page must be the address of the variable that was returned
 * from ring_buffer_alloc_read_page. This is because the page might be used
 * to swap with a page in the ring buffer.
 *
 * for example:
 *        rpage = ring_buffer_alloc_read_page(buffer, cpu);
 *        if (IS_ERR(rpage))
 *                return PTR_ERR(rpage);
 *        ret = ring_buffer_read_page(buffer, rpage, len, cpu, 0);
 *        if (ret >= 0)
 *                process_page(ring_buffer_read_page_data(rpage), ret);
 *        ring_buffer_free_read_page(buffer, cpu, rpage);
 *
 * When @full is set, the function will not return true unless
 * the writer is off the reader page.
 *
 * Note: it is up to the calling functions to handle sleeps and wakeups.
 *  The ring buffer can be used anywhere in the kernel and can not
 *  blindly call wake_up. The layer that uses the ring buffer must be
 *  responsible for that.
 *
 * Returns:
 *  >=0 if data has been transferred, returns the offset of consumed data.
 *  <0 if no data has been transferred.
 */
int ring_buffer_read_page(struct trace_buffer *buffer,
                          struct buffer_data_read_page *data_page,
                          size_t len, int cpu, int full)
{
        struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
        struct ring_buffer_event *event;
        struct buffer_data_page *bpage;
        struct buffer_page *reader;
        unsigned long missed_events;
        unsigned long flags;
        unsigned int commit;
        unsigned int read;
        u64 save_timestamp;
        int ret = -1;

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                goto out;

        /*
         * If len is not big enough to hold the page header, then
         * we can not copy anything.
         */
        if (len <= BUF_PAGE_HDR_SIZE)
                goto out;

        len -= BUF_PAGE_HDR_SIZE;

        if (!data_page || !data_page->data)
                goto out;
        if (data_page->order != buffer->subbuf_order)
                goto out;

        bpage = data_page->data;
        if (!bpage)
                goto out;

        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);

        reader = rb_get_reader_page(cpu_buffer);
        if (!reader)
                goto out_unlock;

        event = rb_reader_event(cpu_buffer);

        read = reader->read;
        commit = rb_page_size(reader);

        /* Check if any events were dropped */
        missed_events = cpu_buffer->lost_events;

        /*
         * If this page has been partially read or
         * if len is not big enough to read the rest of the page or
         * a writer is still on the page, then
         * we must copy the data from the page to the buffer.
         * Otherwise, we can simply swap the page with the one passed in.
         */
        if (read || (len < (commit - read)) ||
            cpu_buffer->reader_page == cpu_buffer->commit_page ||
            cpu_buffer->mapped) {
                struct buffer_data_page *rpage = cpu_buffer->reader_page->page;
                unsigned int rpos = read;
                unsigned int pos = 0;
                unsigned int size;

                /*
                 * If a full page is expected, this can still be returned
                 * if there's been a previous partial read and the
                 * rest of the page can be read and the commit page is off
                 * the reader page.
                 */
                if (full &&
                    (!read || (len < (commit - read)) ||
                     cpu_buffer->reader_page == cpu_buffer->commit_page))
                        goto out_unlock;

                if (len > (commit - read))
                        len = (commit - read);

                /* Always keep the time extend and data together */
                size = rb_event_ts_length(event);

                if (len < size)
                        goto out_unlock;

                /* save the current timestamp, since the user will need it */
                save_timestamp = cpu_buffer->read_stamp;

                /* Need to copy one event at a time */
                do {
                        /* We need the size of one event, because
                         * rb_advance_reader only advances by one event,
                         * whereas rb_event_ts_length may include the size of
                         * one or two events.
                         * We have already ensured there's enough space if this
                         * is a time extend. */
                        size = rb_event_length(event);
                        memcpy(bpage->data + pos, rpage->data + rpos, size);

                        len -= size;

                        rb_advance_reader(cpu_buffer);
                        rpos = reader->read;
                        pos += size;

                        if (rpos >= commit)
                                break;

                        event = rb_reader_event(cpu_buffer);
                        /* Always keep the time extend and data together */
                        size = rb_event_ts_length(event);
                } while (len >= size);

                /* update bpage */
                local_set(&bpage->commit, pos);
                bpage->time_stamp = save_timestamp;

                /* we copied everything to the beginning */
                read = 0;
        } else {
                /* update the entry counter */
                cpu_buffer->read += rb_page_entries(reader);
                cpu_buffer->read_bytes += rb_page_size(reader);

                /* swap the pages */
                rb_init_page(bpage);
                bpage = reader->page;
                reader->page = data_page->data;
                local_set(&reader->write, 0);
                local_set(&reader->entries, 0);
                reader->read = 0;
                data_page->data = bpage;

                /*
                 * Use the real_end for the data size,
                 * This gives us a chance to store the lost events
                 * on the page.
                 */
                if (reader->real_end)
                        local_set(&bpage->commit, reader->real_end);
        }
        ret = read;

        cpu_buffer->lost_events = 0;

        commit = local_read(&bpage->commit);
        /*
         * Set a flag in the commit field if we lost events
         */
        if (missed_events) {
                /* If there is room at the end of the page to save the
                 * missed events, then record it there.
                 */
                if (buffer->subbuf_size - commit >= sizeof(missed_events)) {
                        memcpy(&bpage->data[commit], &missed_events,
                               sizeof(missed_events));
                        local_add(RB_MISSED_STORED, &bpage->commit);
                        commit += sizeof(missed_events);
                }
                local_add(RB_MISSED_EVENTS, &bpage->commit);
        }

        /*
         * This page may be off to user land. Zero it out here.
         */
        if (commit < buffer->subbuf_size)
                memset(&bpage->data[commit], 0, buffer->subbuf_size - commit);

 out_unlock:
        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);

 out:
        return ret;
}
EXPORT_SYMBOL_GPL(ring_buffer_read_page);

/**
 * ring_buffer_read_page_data - get pointer to the data in the page.
 * @page:  the page to get the data from
 *
 * Returns pointer to the actual data in this page.
 */
void *ring_buffer_read_page_data(struct buffer_data_read_page *page)
{
        return page->data;
}
EXPORT_SYMBOL_GPL(ring_buffer_read_page_data);

/**
 * ring_buffer_subbuf_size_get - get size of the sub buffer.
 * @buffer: the buffer to get the sub buffer size from
 *
 * Returns size of the sub buffer, in bytes.
 */
int ring_buffer_subbuf_size_get(struct trace_buffer *buffer)
{
        return buffer->subbuf_size + BUF_PAGE_HDR_SIZE;
}
EXPORT_SYMBOL_GPL(ring_buffer_subbuf_size_get);

/**
 * ring_buffer_subbuf_order_get - get order of system sub pages in one buffer page.
 * @buffer: The ring_buffer to get the system sub page order from
 *
 * By default, one ring buffer sub page equals to one system page. This parameter
 * is configurable, per ring buffer. The size of the ring buffer sub page can be
 * extended, but must be an order of system page size.
 *
 * Returns the order of buffer sub page size, in system pages:
 * 0 means the sub buffer size is 1 system page and so forth.
 * In case of an error < 0 is returned.
 */
int ring_buffer_subbuf_order_get(struct trace_buffer *buffer)
{
        if (!buffer)
                return -EINVAL;

        return buffer->subbuf_order;
}
EXPORT_SYMBOL_GPL(ring_buffer_subbuf_order_get);

/**
 * ring_buffer_subbuf_order_set - set the size of ring buffer sub page.
 * @buffer: The ring_buffer to set the new page size.
 * @order: Order of the system pages in one sub buffer page
 *
 * By default, one ring buffer pages equals to one system page. This API can be
 * used to set new size of the ring buffer page. The size must be order of
 * system page size, that's why the input parameter @order is the order of
 * system pages that are allocated for one ring buffer page:
 *  0 - 1 system page
 *  1 - 2 system pages
 *  3 - 4 system pages
 *  ...
 *
 * Returns 0 on success or < 0 in case of an error.
 */
int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        struct buffer_page *bpage, *tmp;
        int old_order, old_size;
        int nr_pages;
        int psize;
        int err;
        int cpu;

        if (!buffer || order < 0)
                return -EINVAL;

        if (buffer->subbuf_order == order)
                return 0;

        psize = (1 << order) * PAGE_SIZE;
        if (psize <= BUF_PAGE_HDR_SIZE)
                return -EINVAL;

        /* Size of a subbuf cannot be greater than the write counter */
        if (psize > RB_WRITE_MASK + 1)
                return -EINVAL;

        old_order = buffer->subbuf_order;
        old_size = buffer->subbuf_size;

        /* prevent another thread from changing buffer sizes */
        mutex_lock(&buffer->mutex);
        atomic_inc(&buffer->record_disabled);

        /* Make sure all commits have finished */
        synchronize_rcu();

        buffer->subbuf_order = order;
        buffer->subbuf_size = psize - BUF_PAGE_HDR_SIZE;

        /* Make sure all new buffers are allocated, before deleting the old ones */
        for_each_buffer_cpu(buffer, cpu) {

                if (!cpumask_test_cpu(cpu, buffer->cpumask))
                        continue;

                cpu_buffer = buffer->buffers[cpu];

                if (cpu_buffer->mapped) {
                        err = -EBUSY;
                        goto error;
                }

                /* Update the number of pages to match the new size */
                nr_pages = old_size * buffer->buffers[cpu]->nr_pages;
                nr_pages = DIV_ROUND_UP(nr_pages, buffer->subbuf_size);

                /* we need a minimum of two pages */
                if (nr_pages < 2)
                        nr_pages = 2;

                cpu_buffer->nr_pages_to_update = nr_pages;

                /* Include the reader page */
                nr_pages++;

                /* Allocate the new size buffer */
                INIT_LIST_HEAD(&cpu_buffer->new_pages);
                if (__rb_allocate_pages(cpu_buffer, nr_pages,
                                        &cpu_buffer->new_pages)) {
                        /* not enough memory for new pages */
                        err = -ENOMEM;
                        goto error;
                }
        }

        for_each_buffer_cpu(buffer, cpu) {
                struct buffer_data_page *old_free_data_page;
                struct list_head old_pages;
                unsigned long flags;

                if (!cpumask_test_cpu(cpu, buffer->cpumask))
                        continue;

                cpu_buffer = buffer->buffers[cpu];

                raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);

                /* Clear the head bit to make the link list normal to read */
                rb_head_page_deactivate(cpu_buffer);

                /*
                 * Collect buffers from the cpu_buffer pages list and the
                 * reader_page on old_pages, so they can be freed later when not
                 * under a spinlock. The pages list is a linked list with no
                 * head, adding old_pages turns it into a regular list with
                 * old_pages being the head.
                 */
                list_add(&old_pages, cpu_buffer->pages);
                list_add(&cpu_buffer->reader_page->list, &old_pages);

                /* One page was allocated for the reader page */
                cpu_buffer->reader_page = list_entry(cpu_buffer->new_pages.next,
                                                     struct buffer_page, list);
                list_del_init(&cpu_buffer->reader_page->list);

                /* Install the new pages, remove the head from the list */
                cpu_buffer->pages = cpu_buffer->new_pages.next;
                list_del_init(&cpu_buffer->new_pages);
                cpu_buffer->cnt++;

                cpu_buffer->head_page
                        = list_entry(cpu_buffer->pages, struct buffer_page, list);
                cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page;

                cpu_buffer->nr_pages = cpu_buffer->nr_pages_to_update;
                cpu_buffer->nr_pages_to_update = 0;

                old_free_data_page = cpu_buffer->free_page;
                cpu_buffer->free_page = NULL;

                rb_head_page_activate(cpu_buffer);

                raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);

                /* Free old sub buffers */
                list_for_each_entry_safe(bpage, tmp, &old_pages, list) {
                        list_del_init(&bpage->list);
                        free_buffer_page(bpage);
                }
                free_pages((unsigned long)old_free_data_page, old_order);

                rb_check_pages(cpu_buffer);
        }

        atomic_dec(&buffer->record_disabled);
        mutex_unlock(&buffer->mutex);

        return 0;

error:
        buffer->subbuf_order = old_order;
        buffer->subbuf_size = old_size;

        atomic_dec(&buffer->record_disabled);
        mutex_unlock(&buffer->mutex);

        for_each_buffer_cpu(buffer, cpu) {
                cpu_buffer = buffer->buffers[cpu];

                if (!cpu_buffer->nr_pages_to_update)
                        continue;

                list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages, list) {
                        list_del_init(&bpage->list);
                        free_buffer_page(bpage);
                }
        }

        return err;
}
EXPORT_SYMBOL_GPL(ring_buffer_subbuf_order_set);

static int rb_alloc_meta_page(struct ring_buffer_per_cpu *cpu_buffer)
{
        struct page *page;

        if (cpu_buffer->meta_page)
                return 0;

        page = alloc_page(GFP_USER | __GFP_ZERO);
        if (!page)
                return -ENOMEM;

        cpu_buffer->meta_page = page_to_virt(page);

        return 0;
}

static void rb_free_meta_page(struct ring_buffer_per_cpu *cpu_buffer)
{
        unsigned long addr = (unsigned long)cpu_buffer->meta_page;

        free_page(addr);
        cpu_buffer->meta_page = NULL;
}

static void rb_setup_ids_meta_page(struct ring_buffer_per_cpu *cpu_buffer,
                                   unsigned long *subbuf_ids)
{
        struct trace_buffer_meta *meta = cpu_buffer->meta_page;
        unsigned int nr_subbufs = cpu_buffer->nr_pages + 1;
        struct buffer_page *first_subbuf, *subbuf;
        int id = 0;

        subbuf_ids[id] = (unsigned long)cpu_buffer->reader_page->page;
        cpu_buffer->reader_page->id = id++;

        first_subbuf = subbuf = rb_set_head_page(cpu_buffer);
        do {
                if (WARN_ON(id >= nr_subbufs))
                        break;

                subbuf_ids[id] = (unsigned long)subbuf->page;
                subbuf->id = id;

                rb_inc_page(&subbuf);
                id++;
        } while (subbuf != first_subbuf);

        /* install subbuf ID to kern VA translation */
        cpu_buffer->subbuf_ids = subbuf_ids;

        meta->meta_struct_len = sizeof(*meta);
        meta->nr_subbufs = nr_subbufs;
        meta->subbuf_size = cpu_buffer->buffer->subbuf_size + BUF_PAGE_HDR_SIZE;
        meta->meta_page_size = meta->subbuf_size;

        rb_update_meta_page(cpu_buffer);
}

static struct ring_buffer_per_cpu *
rb_get_mapped_buffer(struct trace_buffer *buffer, int cpu)
{
        struct ring_buffer_per_cpu *cpu_buffer;

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                return ERR_PTR(-EINVAL);

        cpu_buffer = buffer->buffers[cpu];

        mutex_lock(&cpu_buffer->mapping_lock);

        if (!cpu_buffer->user_mapped) {
                mutex_unlock(&cpu_buffer->mapping_lock);
                return ERR_PTR(-ENODEV);
        }

        return cpu_buffer;
}

static void rb_put_mapped_buffer(struct ring_buffer_per_cpu *cpu_buffer)
{
        mutex_unlock(&cpu_buffer->mapping_lock);
}

/*
 * Fast-path for rb_buffer_(un)map(). Called whenever the meta-page doesn't need
 * to be set-up or torn-down.
 */
static int __rb_inc_dec_mapped(struct ring_buffer_per_cpu *cpu_buffer,
                               bool inc)
{
        unsigned long flags;

        lockdep_assert_held(&cpu_buffer->mapping_lock);

        /* mapped is always greater or equal to user_mapped */
        if (WARN_ON(cpu_buffer->mapped < cpu_buffer->user_mapped))
                return -EINVAL;

        if (inc && cpu_buffer->mapped == UINT_MAX)
                return -EBUSY;

        if (WARN_ON(!inc && cpu_buffer->user_mapped == 0))
                return -EINVAL;

        mutex_lock(&cpu_buffer->buffer->mutex);
        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);

        if (inc) {
                cpu_buffer->user_mapped++;
                cpu_buffer->mapped++;
        } else {
                cpu_buffer->user_mapped--;
                cpu_buffer->mapped--;
        }

        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
        mutex_unlock(&cpu_buffer->buffer->mutex);

        return 0;
}

/*
 *   +--------------+  pgoff == 0
 *   |   meta page  |
 *   +--------------+  pgoff == 1
 *   | subbuffer 0  |
 *   |              |
 *   +--------------+  pgoff == (1 + (1 << subbuf_order))
 *   | subbuffer 1  |
 *   |              |
 *         ...
 */
#ifdef CONFIG_MMU
static int __rb_map_vma(struct ring_buffer_per_cpu *cpu_buffer,
                        struct vm_area_struct *vma)
{
        unsigned long nr_subbufs, nr_pages, nr_vma_pages, pgoff = vma->vm_pgoff;
        unsigned int subbuf_pages, subbuf_order;
        struct page **pages;
        int p = 0, s = 0;
        int err;

        /* Refuse MP_PRIVATE or writable mappings */
        if (vma->vm_flags & VM_WRITE || vma->vm_flags & VM_EXEC ||
            !(vma->vm_flags & VM_MAYSHARE))
                return -EPERM;

        subbuf_order = cpu_buffer->buffer->subbuf_order;
        subbuf_pages = 1 << subbuf_order;

        if (subbuf_order && pgoff % subbuf_pages)
                return -EINVAL;

        /*
         * Make sure the mapping cannot become writable later. Also tell the VM
         * to not touch these pages (VM_DONTCOPY | VM_DONTEXPAND).
         */
        vm_flags_mod(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP,
                     VM_MAYWRITE);

        lockdep_assert_held(&cpu_buffer->mapping_lock);

        nr_subbufs = cpu_buffer->nr_pages + 1; /* + reader-subbuf */
        nr_pages = ((nr_subbufs + 1) << subbuf_order); /* + meta-page */
        if (nr_pages <= pgoff)
                return -EINVAL;

        nr_pages -= pgoff;

        nr_vma_pages = vma_pages(vma);
        if (!nr_vma_pages || nr_vma_pages > nr_pages)
                return -EINVAL;

        nr_pages = nr_vma_pages;

        pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL);
        if (!pages)
                return -ENOMEM;

        if (!pgoff) {
                unsigned long meta_page_padding;

                pages[p++] = virt_to_page(cpu_buffer->meta_page);

                /*
                 * Pad with the zero-page to align the meta-page with the
                 * sub-buffers.
                 */
                meta_page_padding = subbuf_pages - 1;
                while (meta_page_padding-- && p < nr_pages) {
                        unsigned long __maybe_unused zero_addr =
                                vma->vm_start + (PAGE_SIZE * p);

                        pages[p++] = ZERO_PAGE(zero_addr);
                }
        } else {
                /* Skip the meta-page */
                pgoff -= subbuf_pages;

                s += pgoff / subbuf_pages;
        }

        while (p < nr_pages) {
                struct page *page;
                int off = 0;

                if (WARN_ON_ONCE(s >= nr_subbufs)) {
                        err = -EINVAL;
                        goto out;
                }

                page = virt_to_page((void *)cpu_buffer->subbuf_ids[s]);

                for (; off < (1 << (subbuf_order)); off++, page++) {
                        if (p >= nr_pages)
                                break;

                        pages[p++] = page;
                }
                s++;
        }

        err = vm_insert_pages(vma, vma->vm_start, pages, &nr_pages);

out:
        kfree(pages);

        return err;
}
#else
static int __rb_map_vma(struct ring_buffer_per_cpu *cpu_buffer,
                        struct vm_area_struct *vma)
{
        return -EOPNOTSUPP;
}
#endif

int ring_buffer_map(struct trace_buffer *buffer, int cpu,
                    struct vm_area_struct *vma)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        unsigned long flags, *subbuf_ids;
        int err = 0;

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                return -EINVAL;

        cpu_buffer = buffer->buffers[cpu];

        mutex_lock(&cpu_buffer->mapping_lock);

        if (cpu_buffer->user_mapped) {
                err = __rb_map_vma(cpu_buffer, vma);
                if (!err)
                        err = __rb_inc_dec_mapped(cpu_buffer, true);
                mutex_unlock(&cpu_buffer->mapping_lock);
                return err;
        }

        /* prevent another thread from changing buffer/sub-buffer sizes */
        mutex_lock(&buffer->mutex);

        err = rb_alloc_meta_page(cpu_buffer);
        if (err)
                goto unlock;

        /* subbuf_ids include the reader while nr_pages does not */
        subbuf_ids = kcalloc(cpu_buffer->nr_pages + 1, sizeof(*subbuf_ids), GFP_KERNEL);
        if (!subbuf_ids) {
                rb_free_meta_page(cpu_buffer);
                err = -ENOMEM;
                goto unlock;
        }

        atomic_inc(&cpu_buffer->resize_disabled);

        /*
         * Lock all readers to block any subbuf swap until the subbuf IDs are
         * assigned.
         */
        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
        rb_setup_ids_meta_page(cpu_buffer, subbuf_ids);

        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);

        err = __rb_map_vma(cpu_buffer, vma);
        if (!err) {
                raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
                /* This is the first time it is mapped by user */
                cpu_buffer->mapped++;
                cpu_buffer->user_mapped = 1;
                raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
        } else {
                kfree(cpu_buffer->subbuf_ids);
                cpu_buffer->subbuf_ids = NULL;
                rb_free_meta_page(cpu_buffer);
                atomic_dec(&cpu_buffer->resize_disabled);
        }

unlock:
        mutex_unlock(&buffer->mutex);
        mutex_unlock(&cpu_buffer->mapping_lock);

        return err;
}

int ring_buffer_unmap(struct trace_buffer *buffer, int cpu)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        unsigned long flags;
        int err = 0;

        if (!cpumask_test_cpu(cpu, buffer->cpumask))
                return -EINVAL;

        cpu_buffer = buffer->buffers[cpu];

        mutex_lock(&cpu_buffer->mapping_lock);

        if (!cpu_buffer->user_mapped) {
                err = -ENODEV;
                goto out;
        } else if (cpu_buffer->user_mapped > 1) {
                __rb_inc_dec_mapped(cpu_buffer, false);
                goto out;
        }

        mutex_lock(&buffer->mutex);
        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);

        /* This is the last user space mapping */
        if (!WARN_ON_ONCE(cpu_buffer->mapped < cpu_buffer->user_mapped))
                cpu_buffer->mapped--;
        cpu_buffer->user_mapped = 0;

        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);

        kfree(cpu_buffer->subbuf_ids);
        cpu_buffer->subbuf_ids = NULL;
        rb_free_meta_page(cpu_buffer);
        atomic_dec(&cpu_buffer->resize_disabled);

        mutex_unlock(&buffer->mutex);

out:
        mutex_unlock(&cpu_buffer->mapping_lock);

        return err;
}

int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu)
{
        struct ring_buffer_per_cpu *cpu_buffer;
        struct buffer_page *reader;
        unsigned long missed_events;
        unsigned long reader_size;
        unsigned long flags;

        cpu_buffer = rb_get_mapped_buffer(buffer, cpu);
        if (IS_ERR(cpu_buffer))
                return (int)PTR_ERR(cpu_buffer);

        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);

consume:
        if (rb_per_cpu_empty(cpu_buffer))
                goto out;

        reader_size = rb_page_size(cpu_buffer->reader_page);

        /*
         * There are data to be read on the current reader page, we can
         * return to the caller. But before that, we assume the latter will read
         * everything. Let's update the kernel reader accordingly.
         */
        if (cpu_buffer->reader_page->read < reader_size) {
                while (cpu_buffer->reader_page->read < reader_size)
                        rb_advance_reader(cpu_buffer);
                goto out;
        }

        reader = rb_get_reader_page(cpu_buffer);
        if (WARN_ON(!reader))
                goto out;

        /* Check if any events were dropped */
        missed_events = cpu_buffer->lost_events;

        if (cpu_buffer->reader_page != cpu_buffer->commit_page) {
                if (missed_events) {
                        struct buffer_data_page *bpage = reader->page;
                        unsigned int commit;
                        /*
                         * Use the real_end for the data size,
                         * This gives us a chance to store the lost events
                         * on the page.
                         */
                        if (reader->real_end)
                                local_set(&bpage->commit, reader->real_end);
                        /*
                         * If there is room at the end of the page to save the
                         * missed events, then record it there.
                         */
                        commit = rb_page_size(reader);
                        if (buffer->subbuf_size - commit >= sizeof(missed_events)) {
                                memcpy(&bpage->data[commit], &missed_events,
                                       sizeof(missed_events));
                                local_add(RB_MISSED_STORED, &bpage->commit);
                        }
                        local_add(RB_MISSED_EVENTS, &bpage->commit);
                }
        } else {
                /*
                 * There really shouldn't be any missed events if the commit
                 * is on the reader page.
                 */
                WARN_ON_ONCE(missed_events);
        }

        cpu_buffer->lost_events = 0;

        goto consume;

out:
        /* Some archs do not have data cache coherency between kernel and user-space */
        flush_kernel_vmap_range(cpu_buffer->reader_page->page,
                                buffer->subbuf_size + BUF_PAGE_HDR_SIZE);

        rb_update_meta_page(cpu_buffer);

        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
        rb_put_mapped_buffer(cpu_buffer);

        return 0;
}

/*
 * We only allocate new buffers, never free them if the CPU goes down.
 * If we were to free the buffer, then the user would lose any trace that was in
 * the buffer.
 */
int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node)
{
        struct trace_buffer *buffer;
        long nr_pages_same;
        int cpu_i;
        unsigned long nr_pages;

        buffer = container_of(node, struct trace_buffer, node);
        if (cpumask_test_cpu(cpu, buffer->cpumask))
                return 0;

        nr_pages = 0;
        nr_pages_same = 1;
        /* check if all cpu sizes are same */
        for_each_buffer_cpu(buffer, cpu_i) {
                /* fill in the size from first enabled cpu */
                if (nr_pages == 0)
                        nr_pages = buffer->buffers[cpu_i]->nr_pages;
                if (nr_pages != buffer->buffers[cpu_i]->nr_pages) {
                        nr_pages_same = 0;
                        break;
                }
        }
        /* allocate minimum pages, user can later expand it */
        if (!nr_pages_same)
                nr_pages = 2;
        buffer->buffers[cpu] =
                rb_allocate_cpu_buffer(buffer, nr_pages, cpu);
        if (!buffer->buffers[cpu]) {
                WARN(1, "failed to allocate ring buffer on CPU %u\n",
                     cpu);
                return -ENOMEM;
        }
        smp_wmb();
        cpumask_set_cpu(cpu, buffer->cpumask);
        return 0;
}

#ifdef CONFIG_RING_BUFFER_STARTUP_TEST
/*
 * This is a basic integrity check of the ring buffer.
 * Late in the boot cycle this test will run when configured in.
 * It will kick off a thread per CPU that will go into a loop
 * writing to the per cpu ring buffer various sizes of data.
 * Some of the data will be large items, some small.
 *
 * Another thread is created that goes into a spin, sending out
 * IPIs to the other CPUs to also write into the ring buffer.
 * this is to test the nesting ability of the buffer.
 *
 * Basic stats are recorded and reported. If something in the
 * ring buffer should happen that's not expected, a big warning
 * is displayed and all ring buffers are disabled.
 */
static struct task_struct *rb_threads[NR_CPUS] __initdata;

struct rb_test_data {
        struct trace_buffer *buffer;
        unsigned long                events;
        unsigned long                bytes_written;
        unsigned long                bytes_alloc;
        unsigned long                bytes_dropped;
        unsigned long                events_nested;
        unsigned long                bytes_written_nested;
        unsigned long                bytes_alloc_nested;
        unsigned long                bytes_dropped_nested;
        int                        min_size_nested;
        int                        max_size_nested;
        int                        max_size;
        int                        min_size;
        int                        cpu;
        int                        cnt;
};

static struct rb_test_data rb_data[NR_CPUS] __initdata;

/* 1 meg per cpu */
#define RB_TEST_BUFFER_SIZE        1048576

static char rb_string[] __initdata =
        "abcdefghijklmnopqrstuvwxyz1234567890!@#$%^&*()?+\\"
        "?+|:';\",.<>/?abcdefghijklmnopqrstuvwxyz1234567890"
        "!@#$%^&*()?+\\?+|:';\",.<>/?abcdefghijklmnopqrstuv";

static bool rb_test_started __initdata;

struct rb_item {
        int size;
        char str[];
};

static __init int rb_write_something(struct rb_test_data *data, bool nested)
{
        struct ring_buffer_event *event;
        struct rb_item *item;
        bool started;
        int event_len;
        int size;
        int len;
        int cnt;

        /* Have nested writes different that what is written */
        cnt = data->cnt + (nested ? 27 : 0);

        /* Multiply cnt by ~e, to make some unique increment */
        size = (cnt * 68 / 25) % (sizeof(rb_string) - 1);

        len = size + sizeof(struct rb_item);

        started = rb_test_started;
        /* read rb_test_started before checking buffer enabled */
        smp_rmb();

        event = ring_buffer_lock_reserve(data->buffer, len);
        if (!event) {
                /* Ignore dropped events before test starts. */
                if (started) {
                        if (nested)
                                data->bytes_dropped_nested += len;
                        else
                                data->bytes_dropped += len;
                }
                return len;
        }

        event_len = ring_buffer_event_length(event);

        if (RB_WARN_ON(data->buffer, event_len < len))
                goto out;

        item = ring_buffer_event_data(event);
        item->size = size;
        memcpy(item->str, rb_string, size);

        if (nested) {
                data->bytes_alloc_nested += event_len;
                data->bytes_written_nested += len;
                data->events_nested++;
                if (!data->min_size_nested || len < data->min_size_nested)
                        data->min_size_nested = len;
                if (len > data->max_size_nested)
                        data->max_size_nested = len;
        } else {
                data->bytes_alloc += event_len;
                data->bytes_written += len;
                data->events++;
                if (!data->min_size || len < data->min_size)
                        data->max_size = len;
                if (len > data->max_size)
                        data->max_size = len;
        }

 out:
        ring_buffer_unlock_commit(data->buffer);

        return 0;
}

static __init int rb_test(void *arg)
{
        struct rb_test_data *data = arg;

        while (!kthread_should_stop()) {
                rb_write_something(data, false);
                data->cnt++;

                set_current_state(TASK_INTERRUPTIBLE);
                /* Now sleep between a min of 100-300us and a max of 1ms */
                usleep_range(((data->cnt % 3) + 1) * 100, 1000);
        }

        return 0;
}

static __init void rb_ipi(void *ignore)
{
        struct rb_test_data *data;
        int cpu = smp_processor_id();

        data = &rb_data[cpu];
        rb_write_something(data, true);
}

static __init int rb_hammer_test(void *arg)
{
        while (!kthread_should_stop()) {

                /* Send an IPI to all cpus to write data! */
                smp_call_function(rb_ipi, NULL, 1);
                /* No sleep, but for non preempt, let others run */
                schedule();
        }

        return 0;
}

static __init int test_ringbuffer(void)
{
        struct task_struct *rb_hammer;
        struct trace_buffer *buffer;
        int cpu;
        int ret = 0;

        if (security_locked_down(LOCKDOWN_TRACEFS)) {
                pr_warn("Lockdown is enabled, skipping ring buffer tests\n");
                return 0;
        }

        pr_info("Running ring buffer tests...\n");

        buffer = ring_buffer_alloc(RB_TEST_BUFFER_SIZE, RB_FL_OVERWRITE);
        if (WARN_ON(!buffer))
                return 0;

        /* Disable buffer so that threads can't write to it yet */
        ring_buffer_record_off(buffer);

        for_each_online_cpu(cpu) {
                rb_data[cpu].buffer = buffer;
                rb_data[cpu].cpu = cpu;
                rb_data[cpu].cnt = cpu;
                rb_threads[cpu] = kthread_run_on_cpu(rb_test, &rb_data[cpu],
                                                     cpu, "rbtester/%u");
                if (WARN_ON(IS_ERR(rb_threads[cpu]))) {
                        pr_cont("FAILED\n");
                        ret = PTR_ERR(rb_threads[cpu]);
                        goto out_free;
                }
        }

        /* Now create the rb hammer! */
        rb_hammer = kthread_run(rb_hammer_test, NULL, "rbhammer");
        if (WARN_ON(IS_ERR(rb_hammer))) {
                pr_cont("FAILED\n");
                ret = PTR_ERR(rb_hammer);
                goto out_free;
        }

        ring_buffer_record_on(buffer);
        /*
         * Show buffer is enabled before setting rb_test_started.
         * Yes there's a small race window where events could be
         * dropped and the thread wont catch it. But when a ring
         * buffer gets enabled, there will always be some kind of
         * delay before other CPUs see it. Thus, we don't care about
         * those dropped events. We care about events dropped after
         * the threads see that the buffer is active.
         */
        smp_wmb();
        rb_test_started = true;

        set_current_state(TASK_INTERRUPTIBLE);
        /* Just run for 10 seconds */;
        schedule_timeout(10 * HZ);

        kthread_stop(rb_hammer);

 out_free:
        for_each_online_cpu(cpu) {
                if (!rb_threads[cpu])
                        break;
                kthread_stop(rb_threads[cpu]);
        }
        if (ret) {
                ring_buffer_free(buffer);
                return ret;
        }

        /* Report! */
        pr_info("finished\n");
        for_each_online_cpu(cpu) {
                struct ring_buffer_event *event;
                struct rb_test_data *data = &rb_data[cpu];
                struct rb_item *item;
                unsigned long total_events;
                unsigned long total_dropped;
                unsigned long total_written;
                unsigned long total_alloc;
                unsigned long total_read = 0;
                unsigned long total_size = 0;
                unsigned long total_len = 0;
                unsigned long total_lost = 0;
                unsigned long lost;
                int big_event_size;
                int small_event_size;

                ret = -1;

                total_events = data->events + data->events_nested;
                total_written = data->bytes_written + data->bytes_written_nested;
                total_alloc = data->bytes_alloc + data->bytes_alloc_nested;
                total_dropped = data->bytes_dropped + data->bytes_dropped_nested;

                big_event_size = data->max_size + data->max_size_nested;
                small_event_size = data->min_size + data->min_size_nested;

                pr_info("CPU %d:\n", cpu);
                pr_info("              events:    %ld\n", total_events);
                pr_info("       dropped bytes:    %ld\n", total_dropped);
                pr_info("       alloced bytes:    %ld\n", total_alloc);
                pr_info("       written bytes:    %ld\n", total_written);
                pr_info("       biggest event:    %d\n", big_event_size);
                pr_info("      smallest event:    %d\n", small_event_size);

                if (RB_WARN_ON(buffer, total_dropped))
                        break;

                ret = 0;

                while ((event = ring_buffer_consume(buffer, cpu, NULL, &lost))) {
                        total_lost += lost;
                        item = ring_buffer_event_data(event);
                        total_len += ring_buffer_event_length(event);
                        total_size += item->size + sizeof(struct rb_item);
                        if (memcmp(&item->str[0], rb_string, item->size) != 0) {
                                pr_info("FAILED!\n");
                                pr_info("buffer had: %.*s\n", item->size, item->str);
                                pr_info("expected:   %.*s\n", item->size, rb_string);
                                RB_WARN_ON(buffer, 1);
                                ret = -1;
                                break;
                        }
                        total_read++;
                }
                if (ret)
                        break;

                ret = -1;

                pr_info("         read events:   %ld\n", total_read);
                pr_info("         lost events:   %ld\n", total_lost);
                pr_info("        total events:   %ld\n", total_lost + total_read);
                pr_info("  recorded len bytes:   %ld\n", total_len);
                pr_info(" recorded size bytes:   %ld\n", total_size);
                if (total_lost) {
                        pr_info(" With dropped events, record len and size may not match\n"
                                " alloced and written from above\n");
                } else {
                        if (RB_WARN_ON(buffer, total_len != total_alloc ||
                                       total_size != total_written))
                                break;
                }
                if (RB_WARN_ON(buffer, total_lost + total_read != total_events))
                        break;

                ret = 0;
        }
        if (!ret)
                pr_info("Ring buffer PASSED!\n");

        ring_buffer_free(buffer);
        return 0;
}

late_initcall(test_ringbuffer);
#endif /* CONFIG_RING_BUFFER_STARTUP_TEST */









































  242 
















  217 
   88 






















  136 




  112 



   47 
    3 



















   48 






  137 




   23 
  129 





    2 
  137 









  113 









































   48 










  133 



    6 

   34 

    1 



  122 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
// SPDX-License-Identifier: GPL-2.0-only
#include <net/tcp.h>

/* The bandwidth estimator estimates the rate at which the network
 * can currently deliver outbound data packets for this flow. At a high
 * level, it operates by taking a delivery rate sample for each ACK.
 *
 * A rate sample records the rate at which the network delivered packets
 * for this flow, calculated over the time interval between the transmission
 * of a data packet and the acknowledgment of that packet.
 *
 * Specifically, over the interval between each transmit and corresponding ACK,
 * the estimator generates a delivery rate sample. Typically it uses the rate
 * at which packets were acknowledged. However, the approach of using only the
 * acknowledgment rate faces a challenge under the prevalent ACK decimation or
 * compression: packets can temporarily appear to be delivered much quicker
 * than the bottleneck rate. Since it is physically impossible to do that in a
 * sustained fashion, when the estimator notices that the ACK rate is faster
 * than the transmit rate, it uses the latter:
 *
 *    send_rate = #pkts_delivered/(last_snd_time - first_snd_time)
 *    ack_rate  = #pkts_delivered/(last_ack_time - first_ack_time)
 *    bw = min(send_rate, ack_rate)
 *
 * Notice the estimator essentially estimates the goodput, not always the
 * network bottleneck link rate when the sending or receiving is limited by
 * other factors like applications or receiver window limits.  The estimator
 * deliberately avoids using the inter-packet spacing approach because that
 * approach requires a large number of samples and sophisticated filtering.
 *
 * TCP flows can often be application-limited in request/response workloads.
 * The estimator marks a bandwidth sample as application-limited if there
 * was some moment during the sampled window of packets when there was no data
 * ready to send in the write queue.
 */

/* Snapshot the current delivery information in the skb, to generate
 * a rate sample later when the skb is (s)acked in tcp_rate_skb_delivered().
 */
void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb)
{
        struct tcp_sock *tp = tcp_sk(sk);

         /* In general we need to start delivery rate samples from the
          * time we received the most recent ACK, to ensure we include
          * the full time the network needs to deliver all in-flight
          * packets. If there are no packets in flight yet, then we
          * know that any ACKs after now indicate that the network was
          * able to deliver those packets completely in the sampling
          * interval between now and the next ACK.
          *
          * Note that we use packets_out instead of tcp_packets_in_flight(tp)
          * because the latter is a guess based on RTO and loss-marking
          * heuristics. We don't want spurious RTOs or loss markings to cause
          * a spuriously small time interval, causing a spuriously high
          * bandwidth estimate.
          */
        if (!tp->packets_out) {
                u64 tstamp_us = tcp_skb_timestamp_us(skb);

                tp->first_tx_mstamp  = tstamp_us;
                tp->delivered_mstamp = tstamp_us;
        }

        TCP_SKB_CB(skb)->tx.first_tx_mstamp        = tp->first_tx_mstamp;
        TCP_SKB_CB(skb)->tx.delivered_mstamp        = tp->delivered_mstamp;
        TCP_SKB_CB(skb)->tx.delivered                = tp->delivered;
        TCP_SKB_CB(skb)->tx.delivered_ce        = tp->delivered_ce;
        TCP_SKB_CB(skb)->tx.is_app_limited        = tp->app_limited ? 1 : 0;
}

/* When an skb is sacked or acked, we fill in the rate sample with the (prior)
 * delivery information when the skb was last transmitted.
 *
 * If an ACK (s)acks multiple skbs (e.g., stretched-acks), this function is
 * called multiple times. We favor the information from the most recently
 * sent skb, i.e., the skb with the most recently sent time and the highest
 * sequence.
 */
void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb,
                            struct rate_sample *rs)
{
        struct tcp_sock *tp = tcp_sk(sk);
        struct tcp_skb_cb *scb = TCP_SKB_CB(skb);
        u64 tx_tstamp;

        if (!scb->tx.delivered_mstamp)
                return;

        tx_tstamp = tcp_skb_timestamp_us(skb);
        if (!rs->prior_delivered ||
            tcp_skb_sent_after(tx_tstamp, tp->first_tx_mstamp,
                               scb->end_seq, rs->last_end_seq)) {
                rs->prior_delivered_ce  = scb->tx.delivered_ce;
                rs->prior_delivered  = scb->tx.delivered;
                rs->prior_mstamp     = scb->tx.delivered_mstamp;
                rs->is_app_limited   = scb->tx.is_app_limited;
                rs->is_retrans             = scb->sacked & TCPCB_RETRANS;
                rs->last_end_seq     = scb->end_seq;

                /* Record send time of most recently ACKed packet: */
                tp->first_tx_mstamp  = tx_tstamp;
                /* Find the duration of the "send phase" of this window: */
                rs->interval_us = tcp_stamp_us_delta(tp->first_tx_mstamp,
                                                     scb->tx.first_tx_mstamp);

        }
        /* Mark off the skb delivered once it's sacked to avoid being
         * used again when it's cumulatively acked. For acked packets
         * we don't need to reset since it'll be freed soon.
         */
        if (scb->sacked & TCPCB_SACKED_ACKED)
                scb->tx.delivered_mstamp = 0;
}

/* Update the connection delivery information and generate a rate sample. */
void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,
                  bool is_sack_reneg, struct rate_sample *rs)
{
        struct tcp_sock *tp = tcp_sk(sk);
        u32 snd_us, ack_us;

        /* Clear app limited if bubble is acked and gone. */
        if (tp->app_limited && after(tp->delivered, tp->app_limited))
                tp->app_limited = 0;

        /* TODO: there are multiple places throughout tcp_ack() to get
         * current time. Refactor the code using a new "tcp_acktag_state"
         * to carry current time, flags, stats like "tcp_sacktag_state".
         */
        if (delivered)
                tp->delivered_mstamp = tp->tcp_mstamp;

        rs->acked_sacked = delivered;        /* freshly ACKed or SACKed */
        rs->losses = lost;                /* freshly marked lost */
        /* Return an invalid sample if no timing information is available or
         * in recovery from loss with SACK reneging. Rate samples taken during
         * a SACK reneging event may overestimate bw by including packets that
         * were SACKed before the reneg.
         */
        if (!rs->prior_mstamp || is_sack_reneg) {
                rs->delivered = -1;
                rs->interval_us = -1;
                return;
        }
        rs->delivered   = tp->delivered - rs->prior_delivered;

        rs->delivered_ce = tp->delivered_ce - rs->prior_delivered_ce;
        /* delivered_ce occupies less than 32 bits in the skb control block */
        rs->delivered_ce &= TCPCB_DELIVERED_CE_MASK;

        /* Model sending data and receiving ACKs as separate pipeline phases
         * for a window. Usually the ACK phase is longer, but with ACK
         * compression the send phase can be longer. To be safe we use the
         * longer phase.
         */
        snd_us = rs->interval_us;                                /* send phase */
        ack_us = tcp_stamp_us_delta(tp->tcp_mstamp,
                                    rs->prior_mstamp); /* ack phase */
        rs->interval_us = max(snd_us, ack_us);

        /* Record both segment send and ack receive intervals */
        rs->snd_interval_us = snd_us;
        rs->rcv_interval_us = ack_us;

        /* Normally we expect interval_us >= min-rtt.
         * Note that rate may still be over-estimated when a spuriously
         * retransmistted skb was first (s)acked because "interval_us"
         * is under-estimated (up to an RTT). However continuously
         * measuring the delivery rate during loss recovery is crucial
         * for connections suffer heavy or prolonged losses.
         */
        if (unlikely(rs->interval_us < tcp_min_rtt(tp))) {
                if (!rs->is_retrans)
                        pr_debug("tcp rate: %ld %d %u %u %u\n",
                                 rs->interval_us, rs->delivered,
                                 inet_csk(sk)->icsk_ca_state,
                                 tp->rx_opt.sack_ok, tcp_min_rtt(tp));
                rs->interval_us = -1;
                return;
        }

        /* Record the last non-app-limited or the highest app-limited bw */
        if (!rs->is_app_limited ||
            ((u64)rs->delivered * tp->rate_interval_us >=
             (u64)tp->rate_delivered * rs->interval_us)) {
                tp->rate_delivered = rs->delivered;
                tp->rate_interval_us = rs->interval_us;
                tp->rate_app_limited = rs->is_app_limited;
        }
}

/* If a gap is detected between sends, mark the socket application-limited. */
void tcp_rate_check_app_limited(struct sock *sk)
{
        struct tcp_sock *tp = tcp_sk(sk);

        if (/* We have less than one packet to send. */
            tp->write_seq - tp->snd_nxt < tp->mss_cache &&
            /* Nothing in sending host's qdisc queues or NIC tx queue. */
            sk_wmem_alloc_get(sk) < SKB_TRUESIZE(1) &&
            /* We are not limited by CWND. */
            tcp_packets_in_flight(tp) < tcp_snd_cwnd(tp) &&
            /* All lost packets have been retransmitted. */
            tp->lost_out <= tp->retrans_out)
                tp->app_limited =
                        (tp->delivered + tcp_packets_in_flight(tp)) ? : 1;
}
EXPORT_SYMBOL_GPL(tcp_rate_check_app_limited);























































































  110 






















































































 1518 








    1 









    1 









    2 

















































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_TIMEKEEPING_H
#define _LINUX_TIMEKEEPING_H

#include <linux/errno.h>
#include <linux/clocksource_ids.h>
#include <linux/ktime.h>

/* Included from linux/ktime.h */

void timekeeping_init(void);
extern int timekeeping_suspended;

/* Architecture timer tick functions: */
extern void legacy_timer_tick(unsigned long ticks);

/*
 * Get and set timeofday
 */
extern int do_settimeofday64(const struct timespec64 *ts);
extern int do_sys_settimeofday64(const struct timespec64 *tv,
                                 const struct timezone *tz);

/*
 * ktime_get() family - read the current time in a multitude of ways.
 *
 * The default time reference is CLOCK_MONOTONIC, starting at
 * boot time but not counting the time spent in suspend.
 * For other references, use the functions with "real", "clocktai",
 * "boottime" and "raw" suffixes.
 *
 * To get the time in a different format, use the ones with
 * "ns", "ts64" and "seconds" suffix.
 *
 * See Documentation/core-api/timekeeping.rst for more details.
 */


/*
 * timespec64 based interfaces
 */
extern void ktime_get_raw_ts64(struct timespec64 *ts);
extern void ktime_get_ts64(struct timespec64 *ts);
extern void ktime_get_real_ts64(struct timespec64 *tv);
extern void ktime_get_coarse_ts64(struct timespec64 *ts);
extern void ktime_get_coarse_real_ts64(struct timespec64 *ts);

/* Multigrain timestamp interfaces */
extern void ktime_get_coarse_real_ts64_mg(struct timespec64 *ts);
extern void ktime_get_real_ts64_mg(struct timespec64 *ts);
extern unsigned long timekeeping_get_mg_floor_swaps(void);

void getboottime64(struct timespec64 *ts);

/*
 * time64_t base interfaces
 */
extern time64_t ktime_get_seconds(void);
extern time64_t __ktime_get_real_seconds(void);
extern time64_t ktime_get_real_seconds(void);

/*
 * ktime_t based interfaces
 */

enum tk_offsets {
        TK_OFFS_REAL,
        TK_OFFS_BOOT,
        TK_OFFS_TAI,
        TK_OFFS_MAX,
};

extern ktime_t ktime_get(void);
extern ktime_t ktime_get_with_offset(enum tk_offsets offs);
extern ktime_t ktime_get_coarse_with_offset(enum tk_offsets offs);
extern ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs);
extern ktime_t ktime_get_raw(void);
extern u32 ktime_get_resolution_ns(void);

/**
 * ktime_get_real - get the real (wall-) time in ktime_t format
 *
 * Returns: real (wall) time in ktime_t format
 */
static inline ktime_t ktime_get_real(void)
{
        return ktime_get_with_offset(TK_OFFS_REAL);
}

static inline ktime_t ktime_get_coarse_real(void)
{
        return ktime_get_coarse_with_offset(TK_OFFS_REAL);
}

/**
 * ktime_get_boottime - Get monotonic time since boot in ktime_t format
 *
 * This is similar to CLOCK_MONTONIC/ktime_get, but also includes the
 * time spent in suspend.
 *
 * Returns: monotonic time since boot in ktime_t format
 */
static inline ktime_t ktime_get_boottime(void)
{
        return ktime_get_with_offset(TK_OFFS_BOOT);
}

static inline ktime_t ktime_get_coarse_boottime(void)
{
        return ktime_get_coarse_with_offset(TK_OFFS_BOOT);
}

/**
 * ktime_get_clocktai - Get the TAI time of day in ktime_t format
 *
 * Returns: the TAI time of day in ktime_t format
 */
static inline ktime_t ktime_get_clocktai(void)
{
        return ktime_get_with_offset(TK_OFFS_TAI);
}

static inline ktime_t ktime_get_coarse_clocktai(void)
{
        return ktime_get_coarse_with_offset(TK_OFFS_TAI);
}

static inline ktime_t ktime_get_coarse(void)
{
        struct timespec64 ts;

        ktime_get_coarse_ts64(&ts);
        return timespec64_to_ktime(ts);
}

static inline u64 ktime_get_coarse_ns(void)
{
        return ktime_to_ns(ktime_get_coarse());
}

static inline u64 ktime_get_coarse_real_ns(void)
{
        return ktime_to_ns(ktime_get_coarse_real());
}

static inline u64 ktime_get_coarse_boottime_ns(void)
{
        return ktime_to_ns(ktime_get_coarse_boottime());
}

static inline u64 ktime_get_coarse_clocktai_ns(void)
{
        return ktime_to_ns(ktime_get_coarse_clocktai());
}

/**
 * ktime_mono_to_real - Convert monotonic time to clock realtime
 * @mono: monotonic time to convert
 *
 * Returns: time converted to realtime clock
 */
static inline ktime_t ktime_mono_to_real(ktime_t mono)
{
        return ktime_mono_to_any(mono, TK_OFFS_REAL);
}

/**
 * ktime_get_ns - Get the current time in nanoseconds
 *
 * Returns: current time converted to nanoseconds
 */
static inline u64 ktime_get_ns(void)
{
        return ktime_to_ns(ktime_get());
}

/**
 * ktime_get_real_ns - Get the current real/wall time in nanoseconds
 *
 * Returns: current real time converted to nanoseconds
 */
static inline u64 ktime_get_real_ns(void)
{
        return ktime_to_ns(ktime_get_real());
}

/**
 * ktime_get_boottime_ns - Get the monotonic time since boot in nanoseconds
 *
 * Returns: current boottime converted to nanoseconds
 */
static inline u64 ktime_get_boottime_ns(void)
{
        return ktime_to_ns(ktime_get_boottime());
}

/**
 * ktime_get_clocktai_ns - Get the current TAI time of day in nanoseconds
 *
 * Returns: current TAI time converted to nanoseconds
 */
static inline u64 ktime_get_clocktai_ns(void)
{
        return ktime_to_ns(ktime_get_clocktai());
}

/**
 * ktime_get_raw_ns - Get the raw monotonic time in nanoseconds
 *
 * Returns: current raw monotonic time converted to nanoseconds
 */
static inline u64 ktime_get_raw_ns(void)
{
        return ktime_to_ns(ktime_get_raw());
}

extern u64 ktime_get_mono_fast_ns(void);
extern u64 ktime_get_raw_fast_ns(void);
extern u64 ktime_get_boot_fast_ns(void);
extern u64 ktime_get_tai_fast_ns(void);
extern u64 ktime_get_real_fast_ns(void);

/*
 * timespec64/time64_t interfaces utilizing the ktime based ones
 * for API completeness, these could be implemented more efficiently
 * if needed.
 */
static inline void ktime_get_boottime_ts64(struct timespec64 *ts)
{
        *ts = ktime_to_timespec64(ktime_get_boottime());
}

static inline void ktime_get_coarse_boottime_ts64(struct timespec64 *ts)
{
        *ts = ktime_to_timespec64(ktime_get_coarse_boottime());
}

static inline time64_t ktime_get_boottime_seconds(void)
{
        return ktime_divns(ktime_get_coarse_boottime(), NSEC_PER_SEC);
}

static inline void ktime_get_clocktai_ts64(struct timespec64 *ts)
{
        *ts = ktime_to_timespec64(ktime_get_clocktai());
}

static inline void ktime_get_coarse_clocktai_ts64(struct timespec64 *ts)
{
        *ts = ktime_to_timespec64(ktime_get_coarse_clocktai());
}

static inline time64_t ktime_get_clocktai_seconds(void)
{
        return ktime_divns(ktime_get_coarse_clocktai(), NSEC_PER_SEC);
}

/*
 * RTC specific
 */
extern bool timekeeping_rtc_skipsuspend(void);
extern bool timekeeping_rtc_skipresume(void);

extern void timekeeping_inject_sleeptime64(const struct timespec64 *delta);

/**
 * struct system_time_snapshot - simultaneous raw/real time capture with
 *                                 counter value
 * @cycles:        Clocksource counter value to produce the system times
 * @real:        Realtime system time
 * @boot:        Boot time
 * @raw:        Monotonic raw system time
 * @cs_id:        Clocksource ID
 * @clock_was_set_seq:        The sequence number of clock-was-set events
 * @cs_was_changed_seq:        The sequence number of clocksource change events
 */
struct system_time_snapshot {
        u64                        cycles;
        ktime_t                        real;
        ktime_t                        boot;
        ktime_t                        raw;
        enum clocksource_ids        cs_id;
        unsigned int                clock_was_set_seq;
        u8                        cs_was_changed_seq;
};

/**
 * struct system_device_crosststamp - system/device cross-timestamp
 *                                      (synchronized capture)
 * @device:                Device time
 * @sys_realtime:        Realtime simultaneous with device time
 * @sys_monoraw:        Monotonic raw simultaneous with device time
 */
struct system_device_crosststamp {
        ktime_t device;
        ktime_t sys_realtime;
        ktime_t sys_monoraw;
};

/**
 * struct system_counterval_t - system counter value with the ID of the
 *                                corresponding clocksource
 * @cycles:        System counter value
 * @cs_id:        Clocksource ID corresponding to system counter value. Used by
 *                timekeeping code to verify comparability of two cycle values.
 *                The default ID, CSID_GENERIC, does not identify a specific
 *                clocksource.
 * @use_nsecs:        @cycles is in nanoseconds.
 */
struct system_counterval_t {
        u64                        cycles;
        enum clocksource_ids        cs_id;
        bool                        use_nsecs;
};

extern bool ktime_real_to_base_clock(ktime_t treal,
                                     enum clocksource_ids base_id, u64 *cycles);
extern bool timekeeping_clocksource_has_base(enum clocksource_ids id);

/*
 * Get cross timestamp between system clock and device clock
 */
extern int get_device_system_crosststamp(
                        int (*get_time_fn)(ktime_t *device_time,
                                struct system_counterval_t *system_counterval,
                                void *ctx),
                        void *ctx,
                        struct system_time_snapshot *history,
                        struct system_device_crosststamp *xtstamp);

/*
 * Simultaneously snapshot realtime and monotonic raw clocks
 */
extern void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot);

/*
 * Persistent clock related interfaces
 */
extern int persistent_clock_is_local;

extern void read_persistent_clock64(struct timespec64 *ts);
void read_persistent_wall_and_boot_offset(struct timespec64 *wall_clock,
                                          struct timespec64 *boot_offset);
#ifdef CONFIG_GENERIC_CMOS_UPDATE
extern int update_persistent_clock64(struct timespec64 now);
#endif

#endif






































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_PROCESSOR_H
#define _ASM_X86_PROCESSOR_H

#include <asm/processor-flags.h>

/* Forward declaration, a strange C thing */
struct task_struct;
struct mm_struct;
struct io_bitmap;
struct vm86;

#include <asm/math_emu.h>
#include <asm/segment.h>
#include <asm/types.h>
#include <uapi/asm/sigcontext.h>
#include <asm/current.h>
#include <asm/cpufeatures.h>
#include <asm/cpuid.h>
#include <asm/page.h>
#include <asm/pgtable_types.h>
#include <asm/percpu.h>
#include <asm/desc_defs.h>
#include <asm/nops.h>
#include <asm/special_insns.h>
#include <asm/fpu/types.h>
#include <asm/unwind_hints.h>
#include <asm/vmxfeatures.h>
#include <asm/vdso/processor.h>
#include <asm/shstk.h>

#include <linux/personality.h>
#include <linux/cache.h>
#include <linux/threads.h>
#include <linux/math64.h>
#include <linux/err.h>
#include <linux/irqflags.h>
#include <linux/mem_encrypt.h>

/*
 * We handle most unaligned accesses in hardware.  On the other hand
 * unaligned DMA can be quite expensive on some Nehalem processors.
 *
 * Based on this we disable the IP header alignment in network drivers.
 */
#define NET_IP_ALIGN        0

#define HBP_NUM 4

/*
 * These alignment constraints are for performance in the vSMP case,
 * but in the task_struct case we must also meet hardware imposed
 * alignment requirements of the FPU state:
 */
#ifdef CONFIG_X86_VSMP
# define ARCH_MIN_TASKALIGN                (1 << INTERNODE_CACHE_SHIFT)
# define ARCH_MIN_MMSTRUCT_ALIGN        (1 << INTERNODE_CACHE_SHIFT)
#else
# define ARCH_MIN_TASKALIGN                __alignof__(union fpregs_state)
# define ARCH_MIN_MMSTRUCT_ALIGN        0
#endif

extern u16 __read_mostly tlb_lli_4k;
extern u16 __read_mostly tlb_lli_2m;
extern u16 __read_mostly tlb_lli_4m;
extern u16 __read_mostly tlb_lld_4k;
extern u16 __read_mostly tlb_lld_2m;
extern u16 __read_mostly tlb_lld_4m;
extern u16 __read_mostly tlb_lld_1g;

/*
 * CPU type and hardware bug flags. Kept separately for each CPU.
 */

struct cpuinfo_topology {
        // Real APIC ID read from the local APIC
        u32                        apicid;
        // The initial APIC ID provided by CPUID
        u32                        initial_apicid;

        // Physical package ID
        u32                        pkg_id;

        // Physical die ID on AMD, Relative on Intel
        u32                        die_id;

        // Compute unit ID - AMD specific
        u32                        cu_id;

        // Core ID relative to the package
        u32                        core_id;

        // Logical ID mappings
        u32                        logical_pkg_id;
        u32                        logical_die_id;
        u32                        logical_core_id;

        // AMD Node ID and Nodes per Package info
        u32                        amd_node_id;

        // Cache level topology IDs
        u32                        llc_id;
        u32                        l2c_id;

        // Hardware defined CPU-type
        union {
                u32                cpu_type;
                struct {
                        // CPUID.1A.EAX[23-0]
                        u32        intel_native_model_id        :24;
                        // CPUID.1A.EAX[31-24]
                        u32        intel_type                :8;
                };
                struct {
                        // CPUID 0x80000026.EBX
                        u32        amd_num_processors        :16,
                                amd_power_eff_ranking        :8,
                                amd_native_model_id        :4,
                                amd_type                :4;
                };
        };
};

struct cpuinfo_x86 {
        union {
                /*
                 * The particular ordering (low-to-high) of (vendor,
                 * family, model) is done in case range of models, like
                 * it is usually done on AMD, need to be compared.
                 */
                struct {
                        __u8        x86_model;
                        /* CPU family */
                        __u8        x86;
                        /* CPU vendor */
                        __u8        x86_vendor;
                        __u8        x86_reserved;
                };
                /* combined vendor, family, model */
                __u32                x86_vfm;
        };
        __u8                        x86_stepping;
#ifdef CONFIG_X86_64
        /* Number of 4K pages in DTLB/ITLB combined(in pages): */
        int                        x86_tlbsize;
#endif
#ifdef CONFIG_X86_VMX_FEATURE_NAMES
        __u32                        vmx_capability[NVMXINTS];
#endif
        __u8                        x86_virt_bits;
        __u8                        x86_phys_bits;
        /* Max extended CPUID function supported: */
        __u32                        extended_cpuid_level;
        /* Maximum supported CPUID level, -1=no CPUID: */
        int                        cpuid_level;
        /*
         * Align to size of unsigned long because the x86_capability array
         * is passed to bitops which require the alignment. Use unnamed
         * union to enforce the array is aligned to size of unsigned long.
         */
        union {
                __u32                x86_capability[NCAPINTS + NBUGINTS];
                unsigned long        x86_capability_alignment;
        };
        char                        x86_vendor_id[16];
        char                        x86_model_id[64];
        struct cpuinfo_topology        topo;
        /* in KB - valid for CPUS which support this call: */
        unsigned int                x86_cache_size;
        int                        x86_cache_alignment;        /* In bytes */
        /* Cache QoS architectural values, valid only on the BSP: */
        int                        x86_cache_max_rmid;        /* max index */
        int                        x86_cache_occ_scale;        /* scale to bytes */
        int                        x86_cache_mbm_width_offset;
        int                        x86_power;
        unsigned long                loops_per_jiffy;
        /* protected processor identification number */
        u64                        ppin;
        u16                        x86_clflush_size;
        /* number of cores as seen by the OS: */
        u16                        booted_cores;
        /* Index into per_cpu list: */
        u16                        cpu_index;
        /*  Is SMT active on this core? */
        bool                        smt_active;
        u32                        microcode;
        /* Address space bits used by the cache internally */
        u8                        x86_cache_bits;
        unsigned                initialized : 1;
} __randomize_layout;

#define X86_VENDOR_INTEL        0
#define X86_VENDOR_CYRIX        1
#define X86_VENDOR_AMD                2
#define X86_VENDOR_UMC                3
#define X86_VENDOR_CENTAUR        5
#define X86_VENDOR_TRANSMETA        7
#define X86_VENDOR_NSC                8
#define X86_VENDOR_HYGON        9
#define X86_VENDOR_ZHAOXIN        10
#define X86_VENDOR_VORTEX        11
#define X86_VENDOR_NUM                12

#define X86_VENDOR_UNKNOWN        0xff

/*
 * capabilities of CPUs
 */
extern struct cpuinfo_x86        boot_cpu_data;
extern struct cpuinfo_x86        new_cpu_data;

extern __u32                        cpu_caps_cleared[NCAPINTS + NBUGINTS];
extern __u32                        cpu_caps_set[NCAPINTS + NBUGINTS];

DECLARE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
#define cpu_data(cpu)                per_cpu(cpu_info, cpu)

extern const struct seq_operations cpuinfo_op;

#define cache_line_size()        (boot_cpu_data.x86_cache_alignment)

extern void cpu_detect(struct cpuinfo_x86 *c);

static inline unsigned long long l1tf_pfn_limit(void)
{
        return BIT_ULL(boot_cpu_data.x86_cache_bits - 1 - PAGE_SHIFT);
}

void init_cpu_devs(void);
void get_cpu_vendor(struct cpuinfo_x86 *c);
extern void early_cpu_init(void);
extern void identify_secondary_cpu(unsigned int cpu);
extern void print_cpu_info(struct cpuinfo_x86 *);
void print_cpu_msr(struct cpuinfo_x86 *);

/*
 * Friendlier CR3 helpers.
 */
static inline unsigned long read_cr3_pa(void)
{
        return __read_cr3() & CR3_ADDR_MASK;
}

static inline unsigned long native_read_cr3_pa(void)
{
        return __native_read_cr3() & CR3_ADDR_MASK;
}

static inline void load_cr3(pgd_t *pgdir)
{
        write_cr3(__sme_pa(pgdir));
}

/*
 * Note that while the legacy 'TSS' name comes from 'Task State Segment',
 * on modern x86 CPUs the TSS also holds information important to 64-bit mode,
 * unrelated to the task-switch mechanism:
 */
#ifdef CONFIG_X86_32
/* This is the TSS defined by the hardware. */
struct x86_hw_tss {
        unsigned short                back_link, __blh;
        unsigned long                sp0;
        unsigned short                ss0, __ss0h;
        unsigned long                sp1;

        /*
         * We don't use ring 1, so ss1 is a convenient scratch space in
         * the same cacheline as sp0.  We use ss1 to cache the value in
         * MSR_IA32_SYSENTER_CS.  When we context switch
         * MSR_IA32_SYSENTER_CS, we first check if the new value being
         * written matches ss1, and, if it's not, then we wrmsr the new
         * value and update ss1.
         *
         * The only reason we context switch MSR_IA32_SYSENTER_CS is
         * that we set it to zero in vm86 tasks to avoid corrupting the
         * stack if we were to go through the sysenter path from vm86
         * mode.
         */
        unsigned short                ss1;        /* MSR_IA32_SYSENTER_CS */

        unsigned short                __ss1h;
        unsigned long                sp2;
        unsigned short                ss2, __ss2h;
        unsigned long                __cr3;
        unsigned long                ip;
        unsigned long                flags;
        unsigned long                ax;
        unsigned long                cx;
        unsigned long                dx;
        unsigned long                bx;
        unsigned long                sp;
        unsigned long                bp;
        unsigned long                si;
        unsigned long                di;
        unsigned short                es, __esh;
        unsigned short                cs, __csh;
        unsigned short                ss, __ssh;
        unsigned short                ds, __dsh;
        unsigned short                fs, __fsh;
        unsigned short                gs, __gsh;
        unsigned short                ldt, __ldth;
        unsigned short                trace;
        unsigned short                io_bitmap_base;

} __attribute__((packed));
#else
struct x86_hw_tss {
        u32                        reserved1;
        u64                        sp0;
        u64                        sp1;

        /*
         * Since Linux does not use ring 2, the 'sp2' slot is unused by
         * hardware.  entry_SYSCALL_64 uses it as scratch space to stash
         * the user RSP value.
         */
        u64                        sp2;

        u64                        reserved2;
        u64                        ist[7];
        u32                        reserved3;
        u32                        reserved4;
        u16                        reserved5;
        u16                        io_bitmap_base;

} __attribute__((packed));
#endif

/*
 * IO-bitmap sizes:
 */
#define IO_BITMAP_BITS                        65536
#define IO_BITMAP_BYTES                        (IO_BITMAP_BITS / BITS_PER_BYTE)
#define IO_BITMAP_LONGS                        (IO_BITMAP_BYTES / sizeof(long))

#define IO_BITMAP_OFFSET_VALID_MAP                                \
        (offsetof(struct tss_struct, io_bitmap.bitmap) -        \
         offsetof(struct tss_struct, x86_tss))

#define IO_BITMAP_OFFSET_VALID_ALL                                \
        (offsetof(struct tss_struct, io_bitmap.mapall) -        \
         offsetof(struct tss_struct, x86_tss))

#ifdef CONFIG_X86_IOPL_IOPERM
/*
 * sizeof(unsigned long) coming from an extra "long" at the end of the
 * iobitmap. The limit is inclusive, i.e. the last valid byte.
 */
# define __KERNEL_TSS_LIMIT        \
        (IO_BITMAP_OFFSET_VALID_ALL + IO_BITMAP_BYTES + \
         sizeof(unsigned long) - 1)
#else
# define __KERNEL_TSS_LIMIT        \
        (offsetof(struct tss_struct, x86_tss) + sizeof(struct x86_hw_tss) - 1)
#endif

/* Base offset outside of TSS_LIMIT so unpriviledged IO causes #GP */
#define IO_BITMAP_OFFSET_INVALID        (__KERNEL_TSS_LIMIT + 1)

struct entry_stack {
        char        stack[PAGE_SIZE];
};

struct entry_stack_page {
        struct entry_stack stack;
} __aligned(PAGE_SIZE);

/*
 * All IO bitmap related data stored in the TSS:
 */
struct x86_io_bitmap {
        /* The sequence number of the last active bitmap. */
        u64                        prev_sequence;

        /*
         * Store the dirty size of the last io bitmap offender. The next
         * one will have to do the cleanup as the switch out to a non io
         * bitmap user will just set x86_tss.io_bitmap_base to a value
         * outside of the TSS limit. So for sane tasks there is no need to
         * actually touch the io_bitmap at all.
         */
        unsigned int                prev_max;

        /*
         * The extra 1 is there because the CPU will access an
         * additional byte beyond the end of the IO permission
         * bitmap. The extra byte must be all 1 bits, and must
         * be within the limit.
         */
        unsigned long                bitmap[IO_BITMAP_LONGS + 1];

        /*
         * Special I/O bitmap to emulate IOPL(3). All bytes zero,
         * except the additional byte at the end.
         */
        unsigned long                mapall[IO_BITMAP_LONGS + 1];
};

struct tss_struct {
        /*
         * The fixed hardware portion.  This must not cross a page boundary
         * at risk of violating the SDM's advice and potentially triggering
         * errata.
         */
        struct x86_hw_tss        x86_tss;

        struct x86_io_bitmap        io_bitmap;
} __aligned(PAGE_SIZE);

DECLARE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss_rw);

/* Per CPU interrupt stacks */
struct irq_stack {
        char                stack[IRQ_STACK_SIZE];
} __aligned(IRQ_STACK_SIZE);

DECLARE_PER_CPU_CACHE_HOT(struct irq_stack *, hardirq_stack_ptr);
#ifdef CONFIG_X86_64
DECLARE_PER_CPU_CACHE_HOT(bool, hardirq_stack_inuse);
#else
DECLARE_PER_CPU_CACHE_HOT(struct irq_stack *, softirq_stack_ptr);
#endif

DECLARE_PER_CPU_CACHE_HOT(unsigned long, cpu_current_top_of_stack);
/* const-qualified alias provided by the linker. */
DECLARE_PER_CPU_CACHE_HOT(const unsigned long __percpu_seg_override,
                          const_cpu_current_top_of_stack);

#ifdef CONFIG_X86_64
static inline unsigned long cpu_kernelmode_gs_base(int cpu)
{
#ifdef CONFIG_SMP
        return per_cpu_offset(cpu);
#else
        return 0;
#endif
}

extern asmlinkage void entry_SYSCALL32_ignore(void);

/* Save actual FS/GS selectors and bases to current->thread */
void current_save_fsgs(void);
#endif        /* X86_64 */

struct perf_event;

struct thread_struct {
        /* Cached TLS descriptors: */
        struct desc_struct        tls_array[GDT_ENTRY_TLS_ENTRIES];
#ifdef CONFIG_X86_32
        unsigned long                sp0;
#endif
        unsigned long                sp;
#ifdef CONFIG_X86_32
        unsigned long                sysenter_cs;
#else
        unsigned short                es;
        unsigned short                ds;
        unsigned short                fsindex;
        unsigned short                gsindex;
#endif

#ifdef CONFIG_X86_64
        unsigned long                fsbase;
        unsigned long                gsbase;
#else
        /*
         * XXX: this could presumably be unsigned short.  Alternatively,
         * 32-bit kernels could be taught to use fsindex instead.
         */
        unsigned long fs;
        unsigned long gs;
#endif

        /* Save middle states of ptrace breakpoints */
        struct perf_event        *ptrace_bps[HBP_NUM];
        /* Debug status used for traps, single steps, etc... */
        unsigned long           virtual_dr6;
        /* Keep track of the exact dr7 value set by the user */
        unsigned long           ptrace_dr7;
        /* Fault info: */
        unsigned long                cr2;
        unsigned long                trap_nr;
        unsigned long                error_code;
#ifdef CONFIG_VM86
        /* Virtual 86 mode info */
        struct vm86                *vm86;
#endif
        /* IO permissions: */
        struct io_bitmap        *io_bitmap;

        /*
         * IOPL. Privilege level dependent I/O permission which is
         * emulated via the I/O bitmap to prevent user space from disabling
         * interrupts.
         */
        unsigned long                iopl_emul;

        unsigned int                iopl_warn:1;

        /*
         * Protection Keys Register for Userspace.  Loaded immediately on
         * context switch. Store it in thread_struct to avoid a lookup in
         * the tasks's FPU xstate buffer. This value is only valid when a
         * task is scheduled out. For 'current' the authoritative source of
         * PKRU is the hardware itself.
         */
        u32                        pkru;

#ifdef CONFIG_X86_USER_SHADOW_STACK
        unsigned long                features;
        unsigned long                features_locked;

        struct thread_shstk        shstk;
#endif

        /* Floating point and extended processor state */
        struct fpu                fpu;
        /*
         * WARNING: 'fpu' is dynamically-sized.  It *MUST* be at
         * the end.
         */
};

extern void fpu_thread_struct_whitelist(unsigned long *offset, unsigned long *size);

static inline void arch_thread_struct_whitelist(unsigned long *offset,
                                                unsigned long *size)
{
        fpu_thread_struct_whitelist(offset, size);
}

static inline void
native_load_sp0(unsigned long sp0)
{
        this_cpu_write(cpu_tss_rw.x86_tss.sp0, sp0);
}

static __always_inline void native_swapgs(void)
{
#ifdef CONFIG_X86_64
        asm volatile("swapgs" ::: "memory");
#endif
}

static __always_inline unsigned long current_top_of_stack(void)
{
        /*
         *  We can't read directly from tss.sp0: sp0 on x86_32 is special in
         *  and around vm86 mode and sp0 on x86_64 is special because of the
         *  entry trampoline.
         */
        if (IS_ENABLED(CONFIG_USE_X86_SEG_SUPPORT))
                return this_cpu_read_const(const_cpu_current_top_of_stack);

        return this_cpu_read_stable(cpu_current_top_of_stack);
}

static __always_inline bool on_thread_stack(void)
{
        return (unsigned long)(current_top_of_stack() -
                               current_stack_pointer) < THREAD_SIZE;
}

#ifdef CONFIG_PARAVIRT_XXL
#include <asm/paravirt.h>
#else

static inline void load_sp0(unsigned long sp0)
{
        native_load_sp0(sp0);
}

#endif /* CONFIG_PARAVIRT_XXL */

unsigned long __get_wchan(struct task_struct *p);

extern void select_idle_routine(void);
extern void amd_e400_c1e_apic_setup(void);

extern unsigned long                boot_option_idle_override;

enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT,
                         IDLE_POLL};

extern void enable_sep_cpu(void);


/* Defined in head.S */
extern struct desc_ptr                early_gdt_descr;

extern void switch_gdt_and_percpu_base(int);
extern void load_direct_gdt(int);
extern void load_fixmap_gdt(int);
extern void cpu_init(void);
extern void cpu_init_exception_handling(bool boot_cpu);
extern void cpu_init_replace_early_idt(void);
extern void cr4_init(void);

extern void set_task_blockstep(struct task_struct *task, bool on);

/* Boot loader type from the setup header: */
extern int                        bootloader_type;
extern int                        bootloader_version;

extern char                        ignore_fpu_irq;

#define HAVE_ARCH_PICK_MMAP_LAYOUT 1
#define ARCH_HAS_PREFETCHW

#ifdef CONFIG_X86_32
# define BASE_PREFETCH                ""
# define ARCH_HAS_PREFETCH
#else
# define BASE_PREFETCH                "prefetcht0 %1"
#endif

/*
 * Prefetch instructions for Pentium III (+) and AMD Athlon (+)
 *
 * It's not worth to care about 3dnow prefetches for the K6
 * because they are microcoded there and very slow.
 */
static inline void prefetch(const void *x)
{
        alternative_input(BASE_PREFETCH, "prefetchnta %1",
                          X86_FEATURE_XMM,
                          "m" (*(const char *)x));
}

/*
 * 3dnow prefetch to get an exclusive cache line.
 * Useful for spinlocks to avoid one state transition in the
 * cache coherency protocol:
 */
static __always_inline void prefetchw(const void *x)
{
        alternative_input(BASE_PREFETCH, "prefetchw %1",
                          X86_FEATURE_3DNOWPREFETCH,
                          "m" (*(const char *)x));
}

#define TOP_OF_INIT_STACK ((unsigned long)&init_stack + sizeof(init_stack) - \
                           TOP_OF_KERNEL_STACK_PADDING)

#define task_top_of_stack(task) ((unsigned long)(task_pt_regs(task) + 1))

#define task_pt_regs(task) \
({                                                                        \
        unsigned long __ptr = (unsigned long)task_stack_page(task);        \
        __ptr += THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;                \
        ((struct pt_regs *)__ptr) - 1;                                        \
})

#ifdef CONFIG_X86_32
#define INIT_THREAD  {                                                          \
        .sp0                        = TOP_OF_INIT_STACK,                          \
        .sysenter_cs                = __KERNEL_CS,                                  \
}

#else
extern unsigned long __top_init_kernel_stack[];

#define INIT_THREAD {                                                        \
        .sp        = (unsigned long)&__top_init_kernel_stack,                \
}

#endif /* CONFIG_X86_64 */

extern void start_thread(struct pt_regs *regs, unsigned long new_ip,
                                               unsigned long new_sp);

/*
 * This decides where the kernel will search for a free chunk of vm
 * space during mmap's.
 */
#define __TASK_UNMAPPED_BASE(task_size)        (PAGE_ALIGN(task_size / 3))
#define TASK_UNMAPPED_BASE                __TASK_UNMAPPED_BASE(TASK_SIZE_LOW)

#define KSTK_EIP(task)                (task_pt_regs(task)->ip)
#define KSTK_ESP(task)                (task_pt_regs(task)->sp)

/* Get/set a process' ability to use the timestamp counter instruction */
#define GET_TSC_CTL(adr)        get_tsc_mode((adr))
#define SET_TSC_CTL(val)        set_tsc_mode((val))

extern int get_tsc_mode(unsigned long adr);
extern int set_tsc_mode(unsigned int val);

DECLARE_PER_CPU(u64, msr_misc_features_shadow);

static inline u32 per_cpu_llc_id(unsigned int cpu)
{
        return per_cpu(cpu_info.topo.llc_id, cpu);
}

static inline u32 per_cpu_l2c_id(unsigned int cpu)
{
        return per_cpu(cpu_info.topo.l2c_id, cpu);
}

#ifdef CONFIG_CPU_SUP_AMD
/*
 * Issue a DIV 0/1 insn to clear any division data from previous DIV
 * operations.
 */
static __always_inline void amd_clear_divider(void)
{
        asm volatile(ALTERNATIVE("", "div %2\n\t", X86_BUG_DIV0)
                     :: "a" (0), "d" (0), "r" (1));
}

extern void amd_check_microcode(void);
#else
static inline void amd_clear_divider(void)                { }
static inline void amd_check_microcode(void)                { }
#endif

extern unsigned long arch_align_stack(unsigned long sp);
void free_init_pages(const char *what, unsigned long begin, unsigned long end);
extern void free_kernel_image_pages(const char *what, void *begin, void *end);

void default_idle(void);
#ifdef        CONFIG_XEN
bool xen_set_default_idle(void);
#else
#define xen_set_default_idle 0
#endif

void __noreturn stop_this_cpu(void *dummy);
void microcode_check(struct cpuinfo_x86 *prev_info);
void store_cpu_caps(struct cpuinfo_x86 *info);

enum l1tf_mitigations {
        L1TF_MITIGATION_OFF,
        L1TF_MITIGATION_FLUSH_NOWARN,
        L1TF_MITIGATION_FLUSH,
        L1TF_MITIGATION_FLUSH_NOSMT,
        L1TF_MITIGATION_FULL,
        L1TF_MITIGATION_FULL_FORCE
};

extern enum l1tf_mitigations l1tf_mitigation;

enum mds_mitigations {
        MDS_MITIGATION_OFF,
        MDS_MITIGATION_AUTO,
        MDS_MITIGATION_FULL,
        MDS_MITIGATION_VMWERV,
};

extern bool gds_ucode_mitigated(void);

/*
 * Make previous memory operations globally visible before
 * a WRMSR.
 *
 * MFENCE makes writes visible, but only affects load/store
 * instructions.  WRMSR is unfortunately not a load/store
 * instruction and is unaffected by MFENCE.  The LFENCE ensures
 * that the WRMSR is not reordered.
 *
 * Most WRMSRs are full serializing instructions themselves and
 * do not require this barrier.  This is only required for the
 * IA32_TSC_DEADLINE and X2APIC MSRs.
 */
static inline void weak_wrmsr_fence(void)
{
        alternative("mfence; lfence", "", ALT_NOT(X86_FEATURE_APIC_MSRS_FENCE));
}

#endif /* _ASM_X86_PROCESSOR_H */






















 1509 

 1514 




 1281 








































  100 

   99 




   62 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
// SPDX-License-Identifier: GPL-2.0
#include <linux/export.h>
#include <linux/spinlock.h>
#include <linux/atomic.h>

/*
 * This is an implementation of the notion of "decrement a
 * reference count, and return locked if it decremented to zero".
 *
 * NOTE NOTE NOTE! This is _not_ equivalent to
 *
 *        if (atomic_dec_and_test(&atomic)) {
 *                spin_lock(&lock);
 *                return 1;
 *        }
 *        return 0;
 *
 * because the spin-lock and the decrement must be
 * "atomic".
 */
int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
{
        /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
        if (atomic_add_unless(atomic, -1, 1))
                return 0;

        /* Otherwise do it the slow way */
        spin_lock(lock);
        if (atomic_dec_and_test(atomic))
                return 1;
        spin_unlock(lock);
        return 0;
}

EXPORT_SYMBOL(_atomic_dec_and_lock);

int _atomic_dec_and_lock_irqsave(atomic_t *atomic, spinlock_t *lock,
                                 unsigned long *flags)
{
        /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
        if (atomic_add_unless(atomic, -1, 1))
                return 0;

        /* Otherwise do it the slow way */
        spin_lock_irqsave(lock, *flags);
        if (atomic_dec_and_test(atomic))
                return 1;
        spin_unlock_irqrestore(lock, *flags);
        return 0;
}
EXPORT_SYMBOL(_atomic_dec_and_lock_irqsave);

int _atomic_dec_and_raw_lock(atomic_t *atomic, raw_spinlock_t *lock)
{
        /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
        if (atomic_add_unless(atomic, -1, 1))
                return 0;

        /* Otherwise do it the slow way */
        raw_spin_lock(lock);
        if (atomic_dec_and_test(atomic))
                return 1;
        raw_spin_unlock(lock);
        return 0;
}
EXPORT_SYMBOL(_atomic_dec_and_raw_lock);

int _atomic_dec_and_raw_lock_irqsave(atomic_t *atomic, raw_spinlock_t *lock,
                                     unsigned long *flags)
{
        /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
        if (atomic_add_unless(atomic, -1, 1))
                return 0;

        /* Otherwise do it the slow way */
        raw_spin_lock_irqsave(lock, *flags);
        if (atomic_dec_and_test(atomic))
                return 1;
        raw_spin_unlock_irqrestore(lock, *flags);
        return 0;
}
EXPORT_SYMBOL(_atomic_dec_and_raw_lock_irqsave);

























    3 
    3 

    3 
























    9 













    9 

    9 







    9 


















































































































































































































































































































































































































































































































































































































































   16 



















   16 
















    2 






   14 









   17 






















   11 
   11 


   11 





















    2 











    2 
    2 


    2 















   13 











































    4 




    4 







































































































































































































































































































































































































































































































































































































































































































































































































































































































    3 
    3 


























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    1 
    1 








    9 

    1 







    9