Total coverage: 519255 (28%)of 1888267 























    2 




    2 




    2 


    2 





    2 




























    2 

    2 
    2 


    2 









   11 









   11 

   11 
    5 


    5 



    3 


    4 






    2 

    6 
    2 











    4 














    4 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2007-2008 BalaBit IT Ltd.
 * Author: Krisztian Kovacs
 */
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/module.h>
#include <linux/skbuff.h>
#include <net/tcp.h>
#include <net/udp.h>
#include <net/icmp.h>
#include <net/sock.h>
#include <net/inet_sock.h>
#include <net/netfilter/nf_socket.h>
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
#include <net/netfilter/nf_conntrack.h>
#endif

static int
extract_icmp4_fields(const struct sk_buff *skb, u8 *protocol,
                     __be32 *raddr, __be32 *laddr,
                     __be16 *rport, __be16 *lport)
{
        unsigned int outside_hdrlen = ip_hdrlen(skb);
        struct iphdr *inside_iph, _inside_iph;
        struct icmphdr *icmph, _icmph;
        __be16 *ports, _ports[2];

        icmph = skb_header_pointer(skb, outside_hdrlen,
                                   sizeof(_icmph), &_icmph);
        if (icmph == NULL)
                return 1;

        if (!icmp_is_err(icmph->type))
                return 1;

        inside_iph = skb_header_pointer(skb, outside_hdrlen +
                                        sizeof(struct icmphdr),
                                        sizeof(_inside_iph), &_inside_iph);
        if (inside_iph == NULL)
                return 1;

        if (inside_iph->protocol != IPPROTO_TCP &&
            inside_iph->protocol != IPPROTO_UDP)
                return 1;

        ports = skb_header_pointer(skb, outside_hdrlen +
                                   sizeof(struct icmphdr) +
                                   (inside_iph->ihl << 2),
                                   sizeof(_ports), &_ports);
        if (ports == NULL)
                return 1;

        /* the inside IP packet is the one quoted from our side, thus
         * its saddr is the local address */
        *protocol = inside_iph->protocol;
        *laddr = inside_iph->saddr;
        *lport = ports[0];
        *raddr = inside_iph->daddr;
        *rport = ports[1];

        return 0;
}

static struct sock *
nf_socket_get_sock_v4(struct net *net, struct sk_buff *skb, const int doff,
                      const u8 protocol,
                      const __be32 saddr, const __be32 daddr,
                      const __be16 sport, const __be16 dport,
                      const struct net_device *in)
{
        switch (protocol) {
        case IPPROTO_TCP:
                return inet_lookup(net, skb, doff, saddr, sport, daddr, dport,
                                   in->ifindex);
        case IPPROTO_UDP:
                return udp4_lib_lookup(net, saddr, sport, daddr, dport,
                                       in->ifindex);
        }
        return NULL;
}

struct sock *nf_sk_lookup_slow_v4(struct net *net, const struct sk_buff *skb,
                                  const struct net_device *indev)
{
        __be32 daddr, saddr;
        __be16 dport, sport;
        const struct iphdr *iph = ip_hdr(skb);
        struct sk_buff *data_skb = NULL;
        u8 protocol;
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
        enum ip_conntrack_info ctinfo;
        struct nf_conn const *ct;
#endif
        int doff = 0;

        if (ntohs(iph->frag_off) & IP_OFFSET)
                return NULL;

        if (iph->protocol == IPPROTO_UDP || iph->protocol == IPPROTO_TCP) {
                struct tcphdr _hdr;
                struct udphdr *hp;

                hp = skb_header_pointer(skb, ip_hdrlen(skb),
                                        iph->protocol == IPPROTO_UDP ?
                                        sizeof(*hp) : sizeof(_hdr), &_hdr);
                if (hp == NULL)
                        return NULL;

                protocol = iph->protocol;
                saddr = iph->saddr;
                sport = hp->source;
                daddr = iph->daddr;
                dport = hp->dest;
                data_skb = (struct sk_buff *)skb;
                doff = iph->protocol == IPPROTO_TCP ?
                        ip_hdrlen(skb) + __tcp_hdrlen((struct tcphdr *)hp) :
                        ip_hdrlen(skb) + sizeof(*hp);

        } else if (iph->protocol == IPPROTO_ICMP) {
                if (extract_icmp4_fields(skb, &protocol, &saddr, &daddr,
                                         &sport, &dport))
                        return NULL;
        } else {
                return NULL;
        }

#if IS_ENABLED(CONFIG_NF_CONNTRACK)
        /* Do the lookup with the original socket address in
         * case this is a reply packet of an established
         * SNAT-ted connection.
         */
        ct = nf_ct_get(skb, &ctinfo);
        if (ct &&
            ((iph->protocol != IPPROTO_ICMP &&
              ctinfo == IP_CT_ESTABLISHED_REPLY) ||
             (iph->protocol == IPPROTO_ICMP &&
              ctinfo == IP_CT_RELATED_REPLY)) &&
            (ct->status & IPS_SRC_NAT_DONE)) {

                daddr = ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.src.u3.ip;
                dport = (iph->protocol == IPPROTO_TCP) ?
                        ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.src.u.tcp.port :
                        ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.src.u.udp.port;
        }
#endif

        return nf_socket_get_sock_v4(net, data_skb, doff, protocol, saddr,
                                     daddr, sport, dport, indev);
}
EXPORT_SYMBOL_GPL(nf_sk_lookup_slow_v4);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Krisztian Kovacs, Balazs Scheidler");
MODULE_DESCRIPTION("Netfilter IPv4 socket lookup infrastructure");

































































































































































































































  150 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
 * 25-Jul-1998 Major changes to allow for ip chain table
 *
 * 3-Jan-2000 Named tables to allow packet selection for different uses.
 */

/*
 *         Format of an IP firewall descriptor
 *
 *         src, dst, src_mask, dst_mask are always stored in network byte order.
 *         flags are stored in host byte order (of course).
 *         Port numbers are stored in HOST byte order.
 */

#ifndef _UAPI_IPTABLES_H
#define _UAPI_IPTABLES_H

#include <linux/types.h>
#include <linux/compiler.h>
#include <linux/if.h>
#include <linux/netfilter_ipv4.h>

#include <linux/netfilter/x_tables.h>

#ifndef __KERNEL__
#define IPT_FUNCTION_MAXNAMELEN XT_FUNCTION_MAXNAMELEN
#define IPT_TABLE_MAXNAMELEN XT_TABLE_MAXNAMELEN
#define ipt_match xt_match
#define ipt_target xt_target
#define ipt_table xt_table
#define ipt_get_revision xt_get_revision
#define ipt_entry_match xt_entry_match
#define ipt_entry_target xt_entry_target
#define ipt_standard_target xt_standard_target
#define ipt_error_target xt_error_target
#define ipt_counters xt_counters
#define IPT_CONTINUE XT_CONTINUE
#define IPT_RETURN XT_RETURN

/* This group is older than old (iptables < v1.4.0-rc1~89) */
#include <linux/netfilter/xt_tcpudp.h>
#define ipt_udp xt_udp
#define ipt_tcp xt_tcp
#define IPT_TCP_INV_SRCPT        XT_TCP_INV_SRCPT
#define IPT_TCP_INV_DSTPT        XT_TCP_INV_DSTPT
#define IPT_TCP_INV_FLAGS        XT_TCP_INV_FLAGS
#define IPT_TCP_INV_OPTION        XT_TCP_INV_OPTION
#define IPT_TCP_INV_MASK        XT_TCP_INV_MASK
#define IPT_UDP_INV_SRCPT        XT_UDP_INV_SRCPT
#define IPT_UDP_INV_DSTPT        XT_UDP_INV_DSTPT
#define IPT_UDP_INV_MASK        XT_UDP_INV_MASK

/* The argument to IPT_SO_ADD_COUNTERS. */
#define ipt_counters_info xt_counters_info
/* Standard return verdict, or do jump. */
#define IPT_STANDARD_TARGET XT_STANDARD_TARGET
/* Error verdict. */
#define IPT_ERROR_TARGET XT_ERROR_TARGET

/* fn returns 0 to continue iteration */
#define IPT_MATCH_ITERATE(e, fn, args...) \
        XT_MATCH_ITERATE(struct ipt_entry, e, fn, ## args)

/* fn returns 0 to continue iteration */
#define IPT_ENTRY_ITERATE(entries, size, fn, args...) \
        XT_ENTRY_ITERATE(struct ipt_entry, entries, size, fn, ## args)
#endif

/* Yes, Virginia, you have to zero the padding. */
struct ipt_ip {
        /* Source and destination IP addr */
        struct in_addr src, dst;
        /* Mask for src and dest IP addr */
        struct in_addr smsk, dmsk;
        char iniface[IFNAMSIZ], outiface[IFNAMSIZ];
        unsigned char iniface_mask[IFNAMSIZ], outiface_mask[IFNAMSIZ];

        /* Protocol, 0 = ANY */
        __u16 proto;

        /* Flags word */
        __u8 flags;
        /* Inverse flags */
        __u8 invflags;
};

/* Values for "flag" field in struct ipt_ip (general ip structure). */
#define IPT_F_FRAG                0x01        /* Set if rule is a fragment rule */
#define IPT_F_GOTO                0x02        /* Set if jump is a goto */
#define IPT_F_MASK                0x03        /* All possible flag bits mask. */

/* Values for "inv" field in struct ipt_ip. */
#define IPT_INV_VIA_IN                0x01        /* Invert the sense of IN IFACE. */
#define IPT_INV_VIA_OUT                0x02        /* Invert the sense of OUT IFACE */
#define IPT_INV_TOS                0x04        /* Invert the sense of TOS. */
#define IPT_INV_SRCIP                0x08        /* Invert the sense of SRC IP. */
#define IPT_INV_DSTIP                0x10        /* Invert the sense of DST OP. */
#define IPT_INV_FRAG                0x20        /* Invert the sense of FRAG. */
#define IPT_INV_PROTO                XT_INV_PROTO
#define IPT_INV_MASK                0x7F        /* All possible flag bits mask. */

/* This structure defines each of the firewall rules.  Consists of 3
   parts which are 1) general IP header stuff 2) match specific
   stuff 3) the target to perform if the rule matches */
struct ipt_entry {
        struct ipt_ip ip;

        /* Mark with fields that we care about. */
        unsigned int nfcache;

        /* Size of ipt_entry + matches */
        __u16 target_offset;
        /* Size of ipt_entry + matches + target */
        __u16 next_offset;

        /* Back pointer */
        unsigned int comefrom;

        /* Packet and byte counters. */
        struct xt_counters counters;

        /* The matches (if any), then the target. */
        unsigned char elems[];
};

/*
 * New IP firewall options for [gs]etsockopt at the RAW IP level.
 * Unlike BSD Linux inherits IP options so you don't have to use a raw
 * socket for this. Instead we check rights in the calls.
 *
 * ATTENTION: check linux/in.h before adding new number here.
 */
#define IPT_BASE_CTL                64

#define IPT_SO_SET_REPLACE        (IPT_BASE_CTL)
#define IPT_SO_SET_ADD_COUNTERS        (IPT_BASE_CTL + 1)
#define IPT_SO_SET_MAX                IPT_SO_SET_ADD_COUNTERS

#define IPT_SO_GET_INFO                        (IPT_BASE_CTL)
#define IPT_SO_GET_ENTRIES                (IPT_BASE_CTL + 1)
#define IPT_SO_GET_REVISION_MATCH        (IPT_BASE_CTL + 2)
#define IPT_SO_GET_REVISION_TARGET        (IPT_BASE_CTL + 3)
#define IPT_SO_GET_MAX                        IPT_SO_GET_REVISION_TARGET

/* ICMP matching stuff */
struct ipt_icmp {
        __u8 type;                                /* type to match */
        __u8 code[2];                                /* range of code */
        __u8 invflags;                                /* Inverse flags */
};

/* Values for "inv" field for struct ipt_icmp. */
#define IPT_ICMP_INV        0x01        /* Invert the sense of type/code test */

/* The argument to IPT_SO_GET_INFO */
struct ipt_getinfo {
        /* Which table: caller fills this in. */
        char name[XT_TABLE_MAXNAMELEN];

        /* Kernel fills these in. */
        /* Which hook entry points are valid: bitmask */
        unsigned int valid_hooks;

        /* Hook entry points: one per netfilter hook. */
        unsigned int hook_entry[NF_INET_NUMHOOKS];

        /* Underflow points. */
        unsigned int underflow[NF_INET_NUMHOOKS];

        /* Number of entries */
        unsigned int num_entries;

        /* Size of entries. */
        unsigned int size;
};

/* The argument to IPT_SO_SET_REPLACE. */
struct ipt_replace {
        /* Which table. */
        char name[XT_TABLE_MAXNAMELEN];

        /* Which hook entry points are valid: bitmask.  You can't
           change this. */
        unsigned int valid_hooks;

        /* Number of entries */
        unsigned int num_entries;

        /* Total size of new entries */
        unsigned int size;

        /* Hook entry points. */
        unsigned int hook_entry[NF_INET_NUMHOOKS];

        /* Underflow points. */
        unsigned int underflow[NF_INET_NUMHOOKS];

        /* Information about old entries: */
        /* Number of counters (must be equal to current number of entries). */
        unsigned int num_counters;
        /* The old entries' counters. */
        struct xt_counters __user *counters;

        /* The entries (hang off end: not really an array). */
        struct ipt_entry entries[];
};

/* The argument to IPT_SO_GET_ENTRIES. */
struct ipt_get_entries {
        /* Which table: user fills this in. */
        char name[XT_TABLE_MAXNAMELEN];

        /* User fills this in: total entry size. */
        unsigned int size;

        /* The entries. */
        struct ipt_entry entrytable[];
};

/* Helper functions */
static __inline__ struct xt_entry_target *
ipt_get_target(struct ipt_entry *e)
{
        return (struct xt_entry_target *)((char *)e + e->target_offset);
}

/*
 *        Main firewall chains definitions and global var's definitions.
 */
#endif /* _UAPI_IPTABLES_H */


































































































   10 



   10 




   10 
   10 




























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * IPv6 specific functions of netfilter core
 *
 * Rusty Russell (C) 2000
 * Patrick McHardy (C) 2006-2012
 */
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/ipv6.h>
#include <linux/netfilter.h>
#include <linux/netfilter_ipv6.h>
#include <linux/export.h>
#include <net/addrconf.h>
#include <net/dst.h>
#include <net/ipv6.h>
#include <net/ip6_route.h>
#include <net/xfrm.h>
#include <net/netfilter/nf_queue.h>
#include <net/netfilter/nf_conntrack_bridge.h>
#include <net/netfilter/ipv6/nf_defrag_ipv6.h>
#include "../bridge/br_private.h"

int ip6_route_me_harder(struct net *net, struct sock *sk_partial, struct sk_buff *skb)
{
        const struct ipv6hdr *iph = ipv6_hdr(skb);
        struct sock *sk = sk_to_full_sk(sk_partial);
        struct net_device *dev = skb_dst_dev(skb);
        struct flow_keys flkeys;
        unsigned int hh_len;
        struct dst_entry *dst;
        int strict = (ipv6_addr_type(&iph->daddr) &
                      (IPV6_ADDR_MULTICAST | IPV6_ADDR_LINKLOCAL));
        struct flowi6 fl6 = {
                .flowi6_l3mdev = l3mdev_master_ifindex(dev),
                .flowi6_mark = skb->mark,
                .flowi6_uid = sock_net_uid(net, sk),
                .daddr = iph->daddr,
                .saddr = iph->saddr,
                .flowlabel = ip6_flowinfo(iph),
        };
        int err;

        if (sk && sk->sk_bound_dev_if)
                fl6.flowi6_oif = sk->sk_bound_dev_if;
        else if (strict)
                fl6.flowi6_oif = dev->ifindex;

        fib6_rules_early_flow_dissect(net, skb, &fl6, &flkeys);
        dst = ip6_route_output(net, sk, &fl6);
        err = dst->error;
        if (err) {
                IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTNOROUTES);
                net_dbg_ratelimited("ip6_route_me_harder: No more route\n");
                dst_release(dst);
                return err;
        }

        /* Drop old route. */
        skb_dst_drop(skb);

        skb_dst_set(skb, dst);

#ifdef CONFIG_XFRM
        if (!(IP6CB(skb)->flags & IP6SKB_XFRM_TRANSFORMED) &&
            xfrm_decode_session(net, skb, flowi6_to_flowi(&fl6), AF_INET6) == 0) {
                /* ignore return value from skb_dstref_steal, xfrm_lookup takes
                 * care of dropping the refcnt if needed.
                 */
                skb_dstref_steal(skb);
                dst = xfrm_lookup(net, dst, flowi6_to_flowi(&fl6), sk, 0);
                if (IS_ERR(dst))
                        return PTR_ERR(dst);
                skb_dst_set(skb, dst);
        }
#endif

        /* Change in oif may mean change in hh_len. */
        hh_len = skb_dst_dev(skb)->hard_header_len;
        if (skb_headroom(skb) < hh_len &&
            pskb_expand_head(skb, HH_DATA_ALIGN(hh_len - skb_headroom(skb)),
                             0, GFP_ATOMIC))
                return -ENOMEM;

        return 0;
}
EXPORT_SYMBOL(ip6_route_me_harder);

int __nf_ip6_route(struct net *net, struct dst_entry **dst,
                   struct flowi *fl, bool strict)
{
        static const struct ipv6_pinfo fake_pinfo;
        static const struct inet_sock fake_sk = {
                /* makes ip6_route_output set RT6_LOOKUP_F_IFACE: */
                .sk.sk_bound_dev_if = 1,
                .pinet6 = (struct ipv6_pinfo *) &fake_pinfo,
        };
        const void *sk = strict ? &fake_sk : NULL;
        struct dst_entry *result;
        int err;

        result = ip6_route_output(net, sk, &fl->u.ip6);
        err = result->error;
        if (err)
                dst_release(result);
        else
                *dst = result;
        return err;
}
EXPORT_SYMBOL_GPL(__nf_ip6_route);

int br_ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
                    struct nf_bridge_frag_data *data,
                    int (*output)(struct net *, struct sock *sk,
                                  const struct nf_bridge_frag_data *data,
                                  struct sk_buff *))
{
        int frag_max_size = BR_INPUT_SKB_CB(skb)->frag_max_size;
        u8 tstamp_type = skb->tstamp_type;
        ktime_t tstamp = skb->tstamp;
        struct ip6_frag_state state;
        u8 *prevhdr, nexthdr = 0;
        unsigned int mtu, hlen, nexthdr_offset;
        int hroom, err = 0;
        __be32 frag_id;

        err = ip6_find_1stfragopt(skb, &prevhdr);
        if (err < 0)
                goto blackhole;
        hlen = err;
        nexthdr = *prevhdr;
        nexthdr_offset = prevhdr - skb_network_header(skb);

        mtu = skb->dev->mtu;
        if (frag_max_size > mtu ||
            frag_max_size < IPV6_MIN_MTU)
                goto blackhole;

        mtu = frag_max_size;
        if (mtu < hlen + sizeof(struct frag_hdr) + 8)
                goto blackhole;
        mtu -= hlen + sizeof(struct frag_hdr);

        frag_id = ipv6_select_ident(net, &ipv6_hdr(skb)->daddr,
                                    &ipv6_hdr(skb)->saddr);

        if (skb->ip_summed == CHECKSUM_PARTIAL &&
            (err = skb_checksum_help(skb)))
                goto blackhole;

        prevhdr = skb_network_header(skb) + nexthdr_offset;
        hroom = LL_RESERVED_SPACE(skb->dev);
        if (skb_has_frag_list(skb)) {
                unsigned int first_len = skb_pagelen(skb);
                struct ip6_fraglist_iter iter;
                struct sk_buff *frag2;

                if (first_len - hlen > mtu)
                        goto blackhole;

                if (skb_cloned(skb) ||
                    skb_headroom(skb) < (hroom + sizeof(struct frag_hdr)))
                        goto slow_path;

                skb_walk_frags(skb, frag2) {
                        if (frag2->len > mtu)
                                goto blackhole;

                        /* Partially cloned skb? */
                        if (skb_shared(frag2) ||
                            skb_headroom(frag2) < (hlen + hroom + sizeof(struct frag_hdr)))
                                goto slow_path;
                }

                err = ip6_fraglist_init(skb, hlen, prevhdr, nexthdr, frag_id,
                                        &iter);
                if (err < 0)
                        goto blackhole;

                for (;;) {
                        /* Prepare header of the next frame,
                         * before previous one went down.
                         */
                        if (iter.frag)
                                ip6_fraglist_prepare(skb, &iter);

                        skb_set_delivery_time(skb, tstamp, tstamp_type);
                        err = output(net, sk, data, skb);
                        if (err || !iter.frag)
                                break;

                        skb = ip6_fraglist_next(&iter);
                }

                kfree(iter.tmp_hdr);
                if (!err)
                        return 0;

                kfree_skb_list(iter.frag);
                return err;
        }
slow_path:
        /* This is a linearized skbuff, the original geometry is lost for us.
         * This may also be a clone skbuff, we could preserve the geometry for
         * the copies but probably not worth the effort.
         */
        ip6_frag_init(skb, hlen, mtu, skb->dev->needed_tailroom,
                      LL_RESERVED_SPACE(skb->dev), prevhdr, nexthdr, frag_id,
                      &state);

        while (state.left > 0) {
                struct sk_buff *skb2;

                skb2 = ip6_frag_next(skb, &state);
                if (IS_ERR(skb2)) {
                        err = PTR_ERR(skb2);
                        goto blackhole;
                }

                skb_set_delivery_time(skb2, tstamp, tstamp_type);
                err = output(net, sk, data, skb2);
                if (err)
                        goto blackhole;
        }
        consume_skb(skb);
        return err;

blackhole:
        kfree_skb(skb);
        return 0;
}
EXPORT_SYMBOL_GPL(br_ip6_fragment);











































































    3 

    3 
    3 



    3 



    3 




    3 

    3 
    3 


    3 
    3 


    3 
























    1 











    3 













    3 

    3 








    2 



    2 







    3 





    3 











    3 








    4 



    1 













    1 












    1 






    1 



























    5 















    5 








    5 








    5 







    5 






    5 




    1 







    4 



    4 
    3 










    1 








    1 







    1 








    1 


    1 










    1 


    1 

    5 


    5 

    5 











































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Hauppauge HD PVR USB driver
 *
 * Copyright (C) 2001-2004 Greg Kroah-Hartman (greg@kroah.com)
 * Copyright (C) 2008      Janne Grunau (j@jannau.net)
 * Copyright (C) 2008      John Poet
 */

#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/uaccess.h>
#include <linux/atomic.h>
#include <linux/usb.h>
#include <linux/mutex.h>
#include <linux/i2c.h>

#include <linux/videodev2.h>
#include <media/v4l2-dev.h>
#include <media/v4l2-common.h>

#include "hdpvr.h"

static int video_nr[HDPVR_MAX] = {[0 ... (HDPVR_MAX - 1)] = UNSET};
module_param_array(video_nr, int, NULL, 0);
MODULE_PARM_DESC(video_nr, "video device number (-1=Auto)");

/* holds the number of currently registered devices */
static atomic_t dev_nr = ATOMIC_INIT(-1);

int hdpvr_debug;
module_param(hdpvr_debug, int, S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(hdpvr_debug, "enable debugging output");

static uint default_video_input = HDPVR_VIDEO_INPUTS;
module_param(default_video_input, uint, S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(default_video_input, "default video input: 0=Component / 1=S-Video / 2=Composite");

static uint default_audio_input = HDPVR_AUDIO_INPUTS;
module_param(default_audio_input, uint, S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(default_audio_input, "default audio input: 0=RCA back / 1=RCA front / 2=S/PDIF");

static bool boost_audio;
module_param(boost_audio, bool, S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(boost_audio, "boost the audio signal");


/* table of devices that work with this driver */
static const struct usb_device_id hdpvr_table[] = {
        { USB_DEVICE(HD_PVR_VENDOR_ID, HD_PVR_PRODUCT_ID) },
        { USB_DEVICE(HD_PVR_VENDOR_ID, HD_PVR_PRODUCT_ID1) },
        { USB_DEVICE(HD_PVR_VENDOR_ID, HD_PVR_PRODUCT_ID2) },
        { USB_DEVICE(HD_PVR_VENDOR_ID, HD_PVR_PRODUCT_ID3) },
        { USB_DEVICE(HD_PVR_VENDOR_ID, HD_PVR_PRODUCT_ID4) },
        { }                                        /* Terminating entry */
};
MODULE_DEVICE_TABLE(usb, hdpvr_table);


void hdpvr_delete(struct hdpvr_device *dev)
{
        hdpvr_free_buffers(dev);
        usb_put_dev(dev->udev);
}

static void challenge(u8 *bytes)
{
        __le64 *i64P;
        u64 tmp64;
        uint i, idx;

        for (idx = 0; idx < 32; ++idx) {

                if (idx & 0x3)
                        bytes[(idx >> 3) + 3] = bytes[(idx >> 2) & 0x3];

                switch (idx & 0x3) {
                case 0x3:
                        bytes[2] += bytes[3] * 4 + bytes[4] + bytes[5];
                        bytes[4] += bytes[(idx & 0x1) * 2] * 9 + 9;
                        break;
                case 0x1:
                        bytes[0] *= 8;
                        bytes[0] += 7*idx + 4;
                        bytes[6] += bytes[3] * 3;
                        break;
                case 0x0:
                        bytes[3 - (idx >> 3)] = bytes[idx >> 2];
                        bytes[5] += bytes[6] * 3;
                        for (i = 0; i < 3; i++)
                                bytes[3] *= bytes[3] + 1;
                        break;
                case 0x2:
                        for (i = 0; i < 3; i++)
                                bytes[1] *= bytes[6] + 1;
                        for (i = 0; i < 3; i++) {
                                i64P = (__le64 *)bytes;
                                tmp64 = le64_to_cpup(i64P);
                                tmp64 = tmp64 + (tmp64 << (bytes[7] & 0x0f));
                                *i64P = cpu_to_le64(tmp64);
                        }
                        break;
                }
        }
}

/* try to init the device like the windows driver */
static int device_authorization(struct hdpvr_device *dev)
{

        int ret, retval = -ENOMEM;
        char request_type = 0x38, rcv_request = 0x81;
        char *response;

        mutex_lock(&dev->usbc_mutex);
        ret = usb_control_msg(dev->udev,
                              usb_rcvctrlpipe(dev->udev, 0),
                              rcv_request, 0x80 | request_type,
                              0x0400, 0x0003,
                              dev->usbc_buf, 46,
                              10000);
        if (ret != 46) {
                v4l2_err(&dev->v4l2_dev,
                         "unexpected answer of status request, len %d\n", ret);
                goto unlock;
        }
#ifdef HDPVR_DEBUG
        else {
                v4l2_dbg(MSG_INFO, hdpvr_debug, &dev->v4l2_dev,
                         "Status request returned, len %d: %46ph\n",
                         ret, dev->usbc_buf);
        }
#endif

        dev->fw_ver = dev->usbc_buf[1];

        dev->usbc_buf[46] = '\0';
        v4l2_info(&dev->v4l2_dev, "firmware version 0x%x dated %s\n",
                          dev->fw_ver, &dev->usbc_buf[2]);

        if (dev->fw_ver > 0x15) {
                dev->options.brightness        = 0x80;
                dev->options.contrast        = 0x40;
                dev->options.hue        = 0xf;
                dev->options.saturation        = 0x40;
                dev->options.sharpness        = 0x80;
        }

        switch (dev->fw_ver) {
        case HDPVR_FIRMWARE_VERSION:
                dev->flags &= ~HDPVR_FLAG_AC3_CAP;
                break;
        case HDPVR_FIRMWARE_VERSION_AC3:
        case HDPVR_FIRMWARE_VERSION_0X12:
        case HDPVR_FIRMWARE_VERSION_0X15:
        case HDPVR_FIRMWARE_VERSION_0X1E:
                dev->flags |= HDPVR_FLAG_AC3_CAP;
                break;
        default:
                v4l2_info(&dev->v4l2_dev, "untested firmware, the driver might not work.\n");
                if (dev->fw_ver >= HDPVR_FIRMWARE_VERSION_AC3)
                        dev->flags |= HDPVR_FLAG_AC3_CAP;
                else
                        dev->flags &= ~HDPVR_FLAG_AC3_CAP;
        }

        response = dev->usbc_buf+38;
#ifdef HDPVR_DEBUG
        v4l2_dbg(MSG_INFO, hdpvr_debug, &dev->v4l2_dev, "challenge: %8ph\n",
                 response);
#endif
        challenge(response);
#ifdef HDPVR_DEBUG
        v4l2_dbg(MSG_INFO, hdpvr_debug, &dev->v4l2_dev, " response: %8ph\n",
                 response);
#endif

        msleep(100);
        ret = usb_control_msg(dev->udev,
                              usb_sndctrlpipe(dev->udev, 0),
                              0xd1, 0x00 | request_type,
                              0x0000, 0x0000,
                              response, 8,
                              10000);
        v4l2_dbg(MSG_INFO, hdpvr_debug, &dev->v4l2_dev,
                 "magic request returned %d\n", ret);

        retval = ret != 8;
unlock:
        mutex_unlock(&dev->usbc_mutex);
        return retval;
}

static int hdpvr_device_init(struct hdpvr_device *dev)
{
        int ret;
        u8 *buf;

        if (device_authorization(dev))
                return -EACCES;

        /* default options for init */
        hdpvr_set_options(dev);

        /* set filter options */
        mutex_lock(&dev->usbc_mutex);
        buf = dev->usbc_buf;
        buf[0] = 0x03; buf[1] = 0x03; buf[2] = 0x00; buf[3] = 0x00;
        ret = usb_control_msg(dev->udev,
                              usb_sndctrlpipe(dev->udev, 0),
                              0x01, 0x38,
                              CTRL_LOW_PASS_FILTER_VALUE, CTRL_DEFAULT_INDEX,
                              buf, 4,
                              1000);
        v4l2_dbg(MSG_INFO, hdpvr_debug, &dev->v4l2_dev,
                 "control request returned %d\n", ret);
        mutex_unlock(&dev->usbc_mutex);

        /* enable fan and bling leds */
        mutex_lock(&dev->usbc_mutex);
        buf[0] = 0x1;
        ret = usb_control_msg(dev->udev,
                              usb_sndctrlpipe(dev->udev, 0),
                              0xd4, 0x38, 0, 0, buf, 1,
                              1000);
        v4l2_dbg(MSG_INFO, hdpvr_debug, &dev->v4l2_dev,
                 "control request returned %d\n", ret);

        /* boost analog audio */
        buf[0] = boost_audio;
        ret = usb_control_msg(dev->udev,
                              usb_sndctrlpipe(dev->udev, 0),
                              0xd5, 0x38, 0, 0, buf, 1,
                              1000);
        v4l2_dbg(MSG_INFO, hdpvr_debug, &dev->v4l2_dev,
                 "control request returned %d\n", ret);
        mutex_unlock(&dev->usbc_mutex);

        dev->status = STATUS_IDLE;
        return 0;
}

static const struct hdpvr_options hdpvr_default_options = {
        .video_std        = HDPVR_60HZ,
        .video_input        = HDPVR_COMPONENT,
        .audio_input        = HDPVR_RCA_BACK,
        .bitrate        = 65, /* 6 mbps */
        .peak_bitrate        = 90, /* 9 mbps */
        .bitrate_mode        = HDPVR_CONSTANT,
        .gop_mode        = HDPVR_SIMPLE_IDR_GOP,
        .audio_codec        = V4L2_MPEG_AUDIO_ENCODING_AAC,
        /* original picture controls for firmware version <= 0x15 */
        /* updated in device_authorization() for newer firmware */
        .brightness        = 0x86,
        .contrast        = 0x80,
        .hue                = 0x80,
        .saturation        = 0x80,
        .sharpness        = 0x80,
};

static int hdpvr_probe(struct usb_interface *interface,
                       const struct usb_device_id *id)
{
        struct hdpvr_device *dev;
        struct usb_endpoint_descriptor *endpoint;
#if IS_ENABLED(CONFIG_I2C)
        struct i2c_client *client;
#endif
        int dev_num;
        int retval = -ENOMEM;

        /* allocate memory for our device state and initialize it */
        dev = kzalloc_obj(*dev);
        if (!dev) {
                dev_err(&interface->dev, "Out of memory\n");
                goto error;
        }

        /* init video transfer queues first of all */
        /* to prevent oops in hdpvr_delete() on error paths */
        INIT_LIST_HEAD(&dev->free_buff_list);
        INIT_LIST_HEAD(&dev->rec_buff_list);

        /* register v4l2_device early so it can be used for printks */
        if (v4l2_device_register(&interface->dev, &dev->v4l2_dev)) {
                dev_err(&interface->dev, "v4l2_device_register failed\n");
                goto error_free_dev;
        }

        mutex_init(&dev->io_mutex);
        mutex_init(&dev->i2c_mutex);
        mutex_init(&dev->usbc_mutex);
        dev->usbc_buf = kmalloc(64, GFP_KERNEL);
        if (!dev->usbc_buf) {
                v4l2_err(&dev->v4l2_dev, "Out of memory\n");
                goto error_v4l2_unregister;
        }

        init_waitqueue_head(&dev->wait_buffer);
        init_waitqueue_head(&dev->wait_data);

        dev->options = hdpvr_default_options;

        if (default_video_input < HDPVR_VIDEO_INPUTS)
                dev->options.video_input = default_video_input;

        if (default_audio_input < HDPVR_AUDIO_INPUTS) {
                dev->options.audio_input = default_audio_input;
                if (default_audio_input == HDPVR_SPDIF)
                        dev->options.audio_codec =
                                V4L2_MPEG_AUDIO_ENCODING_AC3;
        }

        dev->udev = usb_get_dev(interface_to_usbdev(interface));

        /* set up the endpoint information */
        /* use only the first bulk-in and bulk-out endpoints */
        if (usb_find_bulk_in_endpoint(interface->cur_altsetting, &endpoint)) {
                v4l2_err(&dev->v4l2_dev, "Could not find bulk-in endpoint\n");
                goto error_put_usb;
        }

        /*
         * USB interface description is buggy, reported max packet size is 512
         * bytes, windows driver uses 8192
         */
        dev->bulk_in_size = 8192;
        dev->bulk_in_endpointAddr = endpoint->bEndpointAddress;

        /* init the device */
        if (hdpvr_device_init(dev)) {
                v4l2_err(&dev->v4l2_dev, "device init failed\n");
                goto error_put_usb;
        }

        mutex_lock(&dev->io_mutex);
        if (hdpvr_alloc_buffers(dev, NUM_BUFFERS)) {
                mutex_unlock(&dev->io_mutex);
                v4l2_err(&dev->v4l2_dev,
                         "allocating transfer buffers failed\n");
                goto error_put_usb;
        }
        mutex_unlock(&dev->io_mutex);

#if IS_ENABLED(CONFIG_I2C)
        retval = hdpvr_register_i2c_adapter(dev);
        if (retval < 0) {
                v4l2_err(&dev->v4l2_dev, "i2c adapter register failed\n");
                goto error_free_buffers;
        }

        client = hdpvr_register_ir_i2c(dev);
        if (IS_ERR(client)) {
                v4l2_err(&dev->v4l2_dev, "i2c IR device register failed\n");
                retval = PTR_ERR(client);
                goto reg_fail;
        }
#endif

        dev_num = atomic_inc_return(&dev_nr);
        if (dev_num >= HDPVR_MAX) {
                v4l2_err(&dev->v4l2_dev,
                         "max device number reached, device register failed\n");
                atomic_dec(&dev_nr);
                retval = -ENODEV;
                goto reg_fail;
        }

        retval = hdpvr_register_videodev(dev, &interface->dev,
                                    video_nr[dev_num]);
        if (retval < 0) {
                v4l2_err(&dev->v4l2_dev, "registering videodev failed\n");
                goto reg_fail;
        }

        /* let the user know what node this device is now attached to */
        v4l2_info(&dev->v4l2_dev, "device now attached to %s\n",
                  video_device_node_name(&dev->video_dev));
        return 0;

reg_fail:
#if IS_ENABLED(CONFIG_I2C)
        i2c_del_adapter(&dev->i2c_adapter);
error_free_buffers:
#endif
        hdpvr_free_buffers(dev);
error_put_usb:
        usb_put_dev(dev->udev);
        kfree(dev->usbc_buf);
error_v4l2_unregister:
        v4l2_device_unregister(&dev->v4l2_dev);
error_free_dev:
        kfree(dev);
error:
        return retval;
}

static void hdpvr_disconnect(struct usb_interface *interface)
{
        struct hdpvr_device *dev = to_hdpvr_dev(usb_get_intfdata(interface));

        v4l2_info(&dev->v4l2_dev, "device %s disconnected\n",
                  video_device_node_name(&dev->video_dev));
        /* prevent more I/O from starting and stop any ongoing */
        mutex_lock(&dev->io_mutex);
        dev->status = STATUS_DISCONNECTED;
        wake_up_interruptible(&dev->wait_data);
        wake_up_interruptible(&dev->wait_buffer);
        mutex_unlock(&dev->io_mutex);
        v4l2_device_disconnect(&dev->v4l2_dev);
        msleep(100);
        flush_work(&dev->worker);
        mutex_lock(&dev->io_mutex);
        hdpvr_cancel_queue(dev);
        mutex_unlock(&dev->io_mutex);
#if IS_ENABLED(CONFIG_I2C)
        i2c_del_adapter(&dev->i2c_adapter);
#endif
        video_unregister_device(&dev->video_dev);
        atomic_dec(&dev_nr);
}


static struct usb_driver hdpvr_usb_driver = {
        .name =                "hdpvr",
        .probe =        hdpvr_probe,
        .disconnect =        hdpvr_disconnect,
        .id_table =        hdpvr_table,
};

module_usb_driver(hdpvr_usb_driver);

MODULE_LICENSE("GPL");
MODULE_VERSION("0.2.1");
MODULE_AUTHOR("Janne Grunau");
MODULE_DESCRIPTION("Hauppauge HD PVR driver");
















 3721 


 3721 

    2 



























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
// SPDX-License-Identifier: GPL-2.0
/*
 * Device physical location support
 *
 * Author: Won Chung <wonchung@google.com>
 */

#include <linux/acpi.h>
#include <linux/sysfs.h>
#include <linux/string_choices.h>

#include "physical_location.h"

bool dev_add_physical_location(struct device *dev)
{
        struct acpi_pld_info *pld;

        if (!has_acpi_companion(dev))
                return false;

        if (!acpi_get_physical_device_location(ACPI_HANDLE(dev), &pld))
                return false;

        dev->physical_location = kzalloc_obj(*dev->physical_location);
        if (!dev->physical_location) {
                ACPI_FREE(pld);
                return false;
        }

        dev->physical_location->panel = pld->panel;
        dev->physical_location->vertical_position = pld->vertical_position;
        dev->physical_location->horizontal_position = pld->horizontal_position;
        dev->physical_location->dock = pld->dock;
        dev->physical_location->lid = pld->lid;

        ACPI_FREE(pld);
        return true;
}

static ssize_t panel_show(struct device *dev, struct device_attribute *attr,
        char *buf)
{
        const char *panel;

        switch (dev->physical_location->panel) {
        case DEVICE_PANEL_TOP:
                panel = "top";
                break;
        case DEVICE_PANEL_BOTTOM:
                panel = "bottom";
                break;
        case DEVICE_PANEL_LEFT:
                panel = "left";
                break;
        case DEVICE_PANEL_RIGHT:
                panel = "right";
                break;
        case DEVICE_PANEL_FRONT:
                panel = "front";
                break;
        case DEVICE_PANEL_BACK:
                panel = "back";
                break;
        default:
                panel = "unknown";
        }
        return sysfs_emit(buf, "%s\n", panel);
}
static DEVICE_ATTR_RO(panel);

static ssize_t vertical_position_show(struct device *dev,
        struct device_attribute *attr, char *buf)
{
        const char *vertical_position;

        switch (dev->physical_location->vertical_position) {
        case DEVICE_VERT_POS_UPPER:
                vertical_position = "upper";
                break;
        case DEVICE_VERT_POS_CENTER:
                vertical_position = "center";
                break;
        case DEVICE_VERT_POS_LOWER:
                vertical_position = "lower";
                break;
        default:
                vertical_position = "unknown";
        }
        return sysfs_emit(buf, "%s\n", vertical_position);
}
static DEVICE_ATTR_RO(vertical_position);

static ssize_t horizontal_position_show(struct device *dev,
        struct device_attribute *attr, char *buf)
{
        const char *horizontal_position;

        switch (dev->physical_location->horizontal_position) {
        case DEVICE_HORI_POS_LEFT:
                horizontal_position = "left";
                break;
        case DEVICE_HORI_POS_CENTER:
                horizontal_position = "center";
                break;
        case DEVICE_HORI_POS_RIGHT:
                horizontal_position = "right";
                break;
        default:
                horizontal_position = "unknown";
        }
        return sysfs_emit(buf, "%s\n", horizontal_position);
}
static DEVICE_ATTR_RO(horizontal_position);

static ssize_t dock_show(struct device *dev, struct device_attribute *attr,
        char *buf)
{
        return sysfs_emit(buf, "%s\n",
                str_yes_no(dev->physical_location->dock));
}
static DEVICE_ATTR_RO(dock);

static ssize_t lid_show(struct device *dev, struct device_attribute *attr,
        char *buf)
{
        return sysfs_emit(buf, "%s\n",
                str_yes_no(dev->physical_location->lid));
}
static DEVICE_ATTR_RO(lid);

static struct attribute *dev_attr_physical_location[] = {
        &dev_attr_panel.attr,
        &dev_attr_vertical_position.attr,
        &dev_attr_horizontal_position.attr,
        &dev_attr_dock.attr,
        &dev_attr_lid.attr,
        NULL,
};

const struct attribute_group dev_attr_physical_location_group = {
        .name = "physical_location",
        .attrs = dev_attr_physical_location,
};










  374 


1
2
3
4
5
6
7
8
9
10
11
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NET_NS_HASH_H__
#define __NET_NS_HASH_H__

#include <net/net_namespace.h>

static inline u32 net_hash_mix(const struct net *net)
{
        return net->hash_mix;
}
#endif































   11 

   11 



   11 



   11 

















   11 
    9 


    8 


    8 
    3 






    8 



    8 







    8 
    8 

    7 
    5 







    4 
    4 

    4 









   16 









   16 



   16 
   16 



   10 


    8 


































    1 
    1 






















    1 







    2 







   15 




   15 

    1 




   14 
   14 
    7 
    1 









   13 





   13 


    6 



   15 































    6 
    4 


    3 

    3 



    4 
    3 


    2 


    3 
    2 


    1 


    6 
























    4 
    2 
    1 

    2 
    2 

    1 

    4 







    1 

















   61 






















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
// SPDX-License-Identifier: GPL-2.0-only
/* (C) 1999-2001 Paul `Rusty' Russell
 * (C) 2002-2004 Netfilter Core Team <coreteam@netfilter.org>
 * (C) 2006-2010 Patrick McHardy <kaber@trash.net>
 */

#include <linux/types.h>
#include <linux/timer.h>
#include <linux/netfilter.h>
#include <linux/in.h>
#include <linux/icmp.h>
#include <linux/seq_file.h>
#include <net/ip.h>
#include <net/checksum.h>
#include <linux/netfilter_ipv4.h>
#include <net/netfilter/nf_conntrack_tuple.h>
#include <net/netfilter/nf_conntrack_l4proto.h>
#include <net/netfilter/nf_conntrack_core.h>
#include <net/netfilter/nf_conntrack_timeout.h>
#include <net/netfilter/nf_conntrack_zones.h>
#include <net/netfilter/nf_log.h>

#include "nf_internals.h"

static const unsigned int nf_ct_icmp_timeout = 30*HZ;

bool icmp_pkt_to_tuple(const struct sk_buff *skb, unsigned int dataoff,
                       struct net *net, struct nf_conntrack_tuple *tuple)
{
        const struct icmphdr *hp;
        struct icmphdr _hdr;

        hp = skb_header_pointer(skb, dataoff, sizeof(_hdr), &_hdr);
        if (hp == NULL)
                return false;

        tuple->dst.u.icmp.type = hp->type;
        tuple->src.u.icmp.id = hp->un.echo.id;
        tuple->dst.u.icmp.code = hp->code;

        return true;
}

/* Add 1; spaces filled with 0. */
static const u_int8_t invmap[] = {
        [ICMP_ECHO] = ICMP_ECHOREPLY + 1,
        [ICMP_ECHOREPLY] = ICMP_ECHO + 1,
        [ICMP_TIMESTAMP] = ICMP_TIMESTAMPREPLY + 1,
        [ICMP_TIMESTAMPREPLY] = ICMP_TIMESTAMP + 1,
        [ICMP_INFO_REQUEST] = ICMP_INFO_REPLY + 1,
        [ICMP_INFO_REPLY] = ICMP_INFO_REQUEST + 1,
        [ICMP_ADDRESS] = ICMP_ADDRESSREPLY + 1,
        [ICMP_ADDRESSREPLY] = ICMP_ADDRESS + 1
};

bool nf_conntrack_invert_icmp_tuple(struct nf_conntrack_tuple *tuple,
                                    const struct nf_conntrack_tuple *orig)
{
        if (orig->dst.u.icmp.type >= sizeof(invmap) ||
            !invmap[orig->dst.u.icmp.type])
                return false;

        tuple->src.u.icmp.id = orig->src.u.icmp.id;
        tuple->dst.u.icmp.type = invmap[orig->dst.u.icmp.type] - 1;
        tuple->dst.u.icmp.code = orig->dst.u.icmp.code;
        return true;
}

/* Returns verdict for packet, or -1 for invalid. */
int nf_conntrack_icmp_packet(struct nf_conn *ct,
                             struct sk_buff *skb,
                             enum ip_conntrack_info ctinfo,
                             const struct nf_hook_state *state)
{
        /* Do not immediately delete the connection after the first
           successful reply to avoid excessive conntrackd traffic
           and also to handle correctly ICMP echo reply duplicates. */
        unsigned int *timeout = nf_ct_timeout_lookup(ct);
        static const u_int8_t valid_new[] = {
                [ICMP_ECHO] = 1,
                [ICMP_TIMESTAMP] = 1,
                [ICMP_INFO_REQUEST] = 1,
                [ICMP_ADDRESS] = 1
        };

        if (state->pf != NFPROTO_IPV4)
                return -NF_ACCEPT;

        if (ct->tuplehash[0].tuple.dst.u.icmp.type >= sizeof(valid_new) ||
            !valid_new[ct->tuplehash[0].tuple.dst.u.icmp.type]) {
                /* Can't create a new ICMP `conn' with this. */
                pr_debug("icmp: can't create new conn with type %u\n",
                         ct->tuplehash[0].tuple.dst.u.icmp.type);
                nf_ct_dump_tuple_ip(&ct->tuplehash[0].tuple);
                return -NF_ACCEPT;
        }

        if (!timeout)
                timeout = &nf_icmp_pernet(nf_ct_net(ct))->timeout;

        nf_ct_refresh_acct(ct, ctinfo, skb, *timeout);
        return NF_ACCEPT;
}

/* Check inner header is related to any of the existing connections */
int nf_conntrack_inet_error(struct nf_conn *tmpl, struct sk_buff *skb,
                            unsigned int dataoff,
                            const struct nf_hook_state *state,
                            u8 l4proto, union nf_inet_addr *outer_daddr)
{
        struct nf_conntrack_tuple innertuple, origtuple;
        const struct nf_conntrack_tuple_hash *h;
        const struct nf_conntrack_zone *zone;
        enum ip_conntrack_info ctinfo;
        struct nf_conntrack_zone tmp;
        union nf_inet_addr *ct_daddr;
        enum ip_conntrack_dir dir;
        struct nf_conn *ct;

        WARN_ON(skb_nfct(skb));
        zone = nf_ct_zone_tmpl(tmpl, skb, &tmp);

        /* Are they talking about one of our connections? */
        if (!nf_ct_get_tuplepr(skb, dataoff,
                               state->pf, state->net, &origtuple))
                return -NF_ACCEPT;

        /* Ordinarily, we'd expect the inverted tupleproto, but it's
           been preserved inside the ICMP. */
        if (!nf_ct_invert_tuple(&innertuple, &origtuple))
                return -NF_ACCEPT;

        h = nf_conntrack_find_get(state->net, zone, &innertuple);
        if (!h)
                return -NF_ACCEPT;

        /* Consider: A -> T (=This machine) -> B
         *   Conntrack entry will look like this:
         *      Original:  A->B
         *      Reply:     B->T (SNAT case) OR A
         *
         * When this function runs, we got packet that looks like this:
         * iphdr|icmphdr|inner_iphdr|l4header (tcp, udp, ..).
         *
         * Above nf_conntrack_find_get() makes lookup based on inner_hdr,
         * so we should expect that destination of the found connection
         * matches outer header destination address.
         *
         * In above example, we can consider these two cases:
         *  1. Error coming in reply direction from B or M (middle box) to
         *     T (SNAT case) or A.
         *     Inner saddr will be B, dst will be T or A.
         *     The found conntrack will be reply tuple (B->T/A).
         *  2. Error coming in original direction from A or M to B.
         *     Inner saddr will be A, inner daddr will be B.
         *     The found conntrack will be original tuple (A->B).
         *
         * In both cases, conntrack[dir].dst == inner.dst.
         *
         * A bogus packet could look like this:
         *   Inner: B->T
         *   Outer: B->X (other machine reachable by T).
         *
         * In this case, lookup yields connection A->B and will
         * set packet from B->X as *RELATED*, even though no connection
         * from X was ever seen.
         */
        ct = nf_ct_tuplehash_to_ctrack(h);
        dir = NF_CT_DIRECTION(h);
        ct_daddr = &ct->tuplehash[dir].tuple.dst.u3;
        if (!nf_inet_addr_cmp(outer_daddr, ct_daddr)) {
                if (state->pf == AF_INET) {
                        nf_l4proto_log_invalid(skb, state,
                                               l4proto,
                                               "outer daddr %pI4 != inner %pI4",
                                               &outer_daddr->ip, &ct_daddr->ip);
                } else if (state->pf == AF_INET6) {
                        nf_l4proto_log_invalid(skb, state,
                                               l4proto,
                                               "outer daddr %pI6 != inner %pI6",
                                               &outer_daddr->ip6, &ct_daddr->ip6);
                }
                nf_ct_put(ct);
                return -NF_ACCEPT;
        }

        ctinfo = IP_CT_RELATED;
        if (dir == IP_CT_DIR_REPLY)
                ctinfo += IP_CT_IS_REPLY;

        /* Update skb to refer to this connection */
        nf_ct_set(skb, ct, ctinfo);
        return NF_ACCEPT;
}

static void icmp_error_log(const struct sk_buff *skb,
                           const struct nf_hook_state *state,
                           const char *msg)
{
        nf_l4proto_log_invalid(skb, state, IPPROTO_ICMP, "%s", msg);
}

/* Small and modified version of icmp_rcv */
int nf_conntrack_icmpv4_error(struct nf_conn *tmpl,
                              struct sk_buff *skb, unsigned int dataoff,
                              const struct nf_hook_state *state)
{
        union nf_inet_addr outer_daddr;
        const struct icmphdr *icmph;
        struct icmphdr _ih;

        /* Not enough header? */
        icmph = skb_header_pointer(skb, dataoff, sizeof(_ih), &_ih);
        if (icmph == NULL) {
                icmp_error_log(skb, state, "short packet");
                return -NF_ACCEPT;
        }

        /* See nf_conntrack_proto_tcp.c */
        if (state->net->ct.sysctl_checksum &&
            state->hook == NF_INET_PRE_ROUTING &&
            nf_ip_checksum(skb, state->hook, dataoff, IPPROTO_ICMP)) {
                icmp_error_log(skb, state, "bad hw icmp checksum");
                return -NF_ACCEPT;
        }

        /*
         *        18 is the highest 'known' ICMP type. Anything else is a mystery
         *
         *        RFC 1122: 3.2.2  Unknown ICMP messages types MUST be silently
         *                  discarded.
         */
        if (icmph->type > NR_ICMP_TYPES) {
                icmp_error_log(skb, state, "invalid icmp type");
                return -NF_ACCEPT;
        }

        /* Need to track icmp error message? */
        if (!icmp_is_err(icmph->type))
                return NF_ACCEPT;

        memset(&outer_daddr, 0, sizeof(outer_daddr));
        outer_daddr.ip = ip_hdr(skb)->daddr;

        dataoff += sizeof(*icmph);
        return nf_conntrack_inet_error(tmpl, skb, dataoff, state,
                                       IPPROTO_ICMP, &outer_daddr);
}

#if IS_ENABLED(CONFIG_NF_CT_NETLINK)

#include <linux/netfilter/nfnetlink.h>
#include <linux/netfilter/nfnetlink_conntrack.h>

static int icmp_tuple_to_nlattr(struct sk_buff *skb,
                                const struct nf_conntrack_tuple *t)
{
        if (nla_put_be16(skb, CTA_PROTO_ICMP_ID, t->src.u.icmp.id) ||
            nla_put_u8(skb, CTA_PROTO_ICMP_TYPE, t->dst.u.icmp.type) ||
            nla_put_u8(skb, CTA_PROTO_ICMP_CODE, t->dst.u.icmp.code))
                goto nla_put_failure;
        return 0;

nla_put_failure:
        return -1;
}

static const struct nla_policy icmp_nla_policy[CTA_PROTO_MAX+1] = {
        [CTA_PROTO_ICMP_TYPE]        = { .type = NLA_U8 },
        [CTA_PROTO_ICMP_CODE]        = { .type = NLA_U8 },
        [CTA_PROTO_ICMP_ID]        = { .type = NLA_U16 },
};

static int icmp_nlattr_to_tuple(struct nlattr *tb[],
                                struct nf_conntrack_tuple *tuple,
                                u_int32_t flags)
{
        if (flags & CTA_FILTER_FLAG(CTA_PROTO_ICMP_TYPE)) {
                if (!tb[CTA_PROTO_ICMP_TYPE])
                        return -EINVAL;

                tuple->dst.u.icmp.type = nla_get_u8(tb[CTA_PROTO_ICMP_TYPE]);
                if (tuple->dst.u.icmp.type >= sizeof(invmap) ||
                    !invmap[tuple->dst.u.icmp.type])
                        return -EINVAL;
        }

        if (flags & CTA_FILTER_FLAG(CTA_PROTO_ICMP_CODE)) {
                if (!tb[CTA_PROTO_ICMP_CODE])
                        return -EINVAL;

                tuple->dst.u.icmp.code = nla_get_u8(tb[CTA_PROTO_ICMP_CODE]);
        }

        if (flags & CTA_FILTER_FLAG(CTA_PROTO_ICMP_ID)) {
                if (!tb[CTA_PROTO_ICMP_ID])
                        return -EINVAL;

                tuple->src.u.icmp.id = nla_get_be16(tb[CTA_PROTO_ICMP_ID]);
        }

        return 0;
}

static unsigned int icmp_nlattr_tuple_size(void)
{
        static unsigned int size __read_mostly;

        if (!size)
                size = nla_policy_len(icmp_nla_policy, CTA_PROTO_MAX + 1);

        return size;
}
#endif

#ifdef CONFIG_NF_CONNTRACK_TIMEOUT

#include <linux/netfilter/nfnetlink.h>
#include <linux/netfilter/nfnetlink_cttimeout.h>

static int icmp_timeout_nlattr_to_obj(struct nlattr *tb[],
                                      struct net *net, void *data)
{
        unsigned int *timeout = data;
        struct nf_icmp_net *in = nf_icmp_pernet(net);

        if (tb[CTA_TIMEOUT_ICMP_TIMEOUT]) {
                if (!timeout)
                        timeout = &in->timeout;
                *timeout =
                        ntohl(nla_get_be32(tb[CTA_TIMEOUT_ICMP_TIMEOUT])) * HZ;
        } else if (timeout) {
                /* Set default ICMP timeout. */
                *timeout = in->timeout;
        }
        return 0;
}

static int
icmp_timeout_obj_to_nlattr(struct sk_buff *skb, const void *data)
{
        const unsigned int *timeout = data;

        if (nla_put_be32(skb, CTA_TIMEOUT_ICMP_TIMEOUT, htonl(*timeout / HZ)))
                goto nla_put_failure;
        return 0;

nla_put_failure:
        return -ENOSPC;
}

static const struct nla_policy
icmp_timeout_nla_policy[CTA_TIMEOUT_ICMP_MAX+1] = {
        [CTA_TIMEOUT_ICMP_TIMEOUT]        = { .type = NLA_U32 },
};
#endif /* CONFIG_NF_CONNTRACK_TIMEOUT */

void nf_conntrack_icmp_init_net(struct net *net)
{
        struct nf_icmp_net *in = nf_icmp_pernet(net);

        in->timeout = nf_ct_icmp_timeout;
}

const struct nf_conntrack_l4proto nf_conntrack_l4proto_icmp =
{
        .l4proto                = IPPROTO_ICMP,
        .allow_clash                = true,
#if IS_ENABLED(CONFIG_NF_CT_NETLINK)
        .tuple_to_nlattr        = icmp_tuple_to_nlattr,
        .nlattr_tuple_size        = icmp_nlattr_tuple_size,
        .nlattr_to_tuple        = icmp_nlattr_to_tuple,
        .nla_policy                = icmp_nla_policy,
#endif
#ifdef CONFIG_NF_CONNTRACK_TIMEOUT
        .ctnl_timeout                = {
                .nlattr_to_obj        = icmp_timeout_nlattr_to_obj,
                .obj_to_nlattr        = icmp_timeout_obj_to_nlattr,
                .nlattr_max        = CTA_TIMEOUT_ICMP_MAX,
                .obj_size        = sizeof(unsigned int),
                .nla_policy        = icmp_timeout_nla_policy,
        },
#endif /* CONFIG_NF_CONNTRACK_TIMEOUT */
};









































































   19 



















































































































































































































  125 












 4660 



























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_DAX_H
#define _LINUX_DAX_H

#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/radix-tree.h>

typedef unsigned long dax_entry_t;

struct dax_device;
struct gendisk;
struct iomap_ops;
struct iomap_iter;
struct iomap;

enum dax_access_mode {
        DAX_ACCESS,
        DAX_RECOVERY_WRITE,
};

struct dax_operations {
        /*
         * direct_access: translate a device-relative
         * logical-page-offset into an absolute physical pfn. Return the
         * number of pages available for DAX at that pfn.
         */
        long (*direct_access)(struct dax_device *, pgoff_t, long,
                        enum dax_access_mode, void **, unsigned long *);
        /* zero_page_range: required operation. Zero page range   */
        int (*zero_page_range)(struct dax_device *, pgoff_t, size_t);
        /*
         * recovery_write: recover a poisoned range by DAX device driver
         * capable of clearing poison.
         */
        size_t (*recovery_write)(struct dax_device *dax_dev, pgoff_t pgoff,
                        void *addr, size_t bytes, struct iov_iter *iter);
};

struct dax_holder_operations {
        /*
         * notify_failure - notify memory failure into inner holder device
         * @dax_dev: the dax device which contains the holder
         * @offset: offset on this dax device where memory failure occurs
         * @len: length of this memory failure event
         * @flags: action flags for memory failure handler
         */
        int (*notify_failure)(struct dax_device *dax_dev, u64 offset,
                        u64 len, int mf_flags);
};

#if IS_ENABLED(CONFIG_DAX)
struct dax_device *alloc_dax(void *private, const struct dax_operations *ops);
void *dax_holder(struct dax_device *dax_dev);
void put_dax(struct dax_device *dax_dev);
void kill_dax(struct dax_device *dax_dev);
struct dax_device *dax_dev_get(dev_t devt);
void dax_write_cache(struct dax_device *dax_dev, bool wc);
bool dax_write_cache_enabled(struct dax_device *dax_dev);
bool dax_synchronous(struct dax_device *dax_dev);
void set_dax_nocache(struct dax_device *dax_dev);
void set_dax_nomc(struct dax_device *dax_dev);
void set_dax_synchronous(struct dax_device *dax_dev);
size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
                void *addr, size_t bytes, struct iov_iter *i);
/*
 * Check if given mapping is supported by the file / underlying device.
 */
static inline bool daxdev_mapping_supported(const struct vm_area_desc *desc,
                                            const struct inode *inode,
                                            struct dax_device *dax_dev)
{
        if (!vma_desc_test(desc, VMA_SYNC_BIT))
                return true;
        if (!IS_DAX(inode))
                return false;
        return dax_synchronous(dax_dev);
}
#else
static inline void *dax_holder(struct dax_device *dax_dev)
{
        return NULL;
}
static inline struct dax_device *alloc_dax(void *private,
                const struct dax_operations *ops)
{
        return ERR_PTR(-EOPNOTSUPP);
}
static inline void put_dax(struct dax_device *dax_dev)
{
}
static inline void kill_dax(struct dax_device *dax_dev)
{
}
static inline void dax_write_cache(struct dax_device *dax_dev, bool wc)
{
}
static inline bool dax_write_cache_enabled(struct dax_device *dax_dev)
{
        return false;
}
static inline bool dax_synchronous(struct dax_device *dax_dev)
{
        return true;
}
static inline void set_dax_nocache(struct dax_device *dax_dev)
{
}
static inline void set_dax_nomc(struct dax_device *dax_dev)
{
}
static inline void set_dax_synchronous(struct dax_device *dax_dev)
{
}
static inline bool daxdev_mapping_supported(const struct vm_area_desc *desc,
                                            const struct inode *inode,
                                            struct dax_device *dax_dev)
{
        return !vma_desc_test(desc, VMA_SYNC_BIT);
}
static inline size_t dax_recovery_write(struct dax_device *dax_dev,
                pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i)
{
        return 0;
}
#endif

struct writeback_control;
#if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX)
int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk);
void dax_remove_host(struct gendisk *disk);
struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off,
                void *holder, const struct dax_holder_operations *ops);
#else
static inline int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk)
{
        return 0;
}
static inline void dax_remove_host(struct gendisk *disk)
{
}
static inline struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev,
                u64 *start_off, void *holder,
                const struct dax_holder_operations *ops)
{
        return NULL;
}
#endif /* CONFIG_BLOCK && CONFIG_FS_DAX */

#if IS_ENABLED(CONFIG_FS_DAX)
void fs_put_dax(struct dax_device *dax_dev, void *holder);
int fs_dax_get(struct dax_device *dax_dev, void *holder,
               const struct dax_holder_operations *hops);
int dax_writeback_mapping_range(struct address_space *mapping,
                struct dax_device *dax_dev, struct writeback_control *wbc);
int dax_folio_reset_order(struct folio *folio);

struct page *dax_layout_busy_page(struct address_space *mapping);
struct page *dax_layout_busy_page_range(struct address_space *mapping, loff_t start, loff_t end);
dax_entry_t dax_lock_folio(struct folio *folio);
void dax_unlock_folio(struct folio *folio, dax_entry_t cookie);
dax_entry_t dax_lock_mapping_entry(struct address_space *mapping,
                unsigned long index, struct page **page);
void dax_unlock_mapping_entry(struct address_space *mapping,
                unsigned long index, dax_entry_t cookie);
#else
static inline void fs_put_dax(struct dax_device *dax_dev, void *holder)
{
}

static inline int fs_dax_get(struct dax_device *dax_dev, void *holder,
                             const struct dax_holder_operations *hops)
{
        return -EOPNOTSUPP;
}
static inline struct page *dax_layout_busy_page(struct address_space *mapping)
{
        return NULL;
}

static inline struct page *dax_layout_busy_page_range(struct address_space *mapping, pgoff_t start, pgoff_t nr_pages)
{
        return NULL;
}

static inline int dax_writeback_mapping_range(struct address_space *mapping,
                struct dax_device *dax_dev, struct writeback_control *wbc)
{
        return -EOPNOTSUPP;
}

static inline dax_entry_t dax_lock_folio(struct folio *folio)
{
        if (IS_DAX(folio->mapping->host))
                return ~0UL;
        return 0;
}

static inline void dax_unlock_folio(struct folio *folio, dax_entry_t cookie)
{
}

static inline dax_entry_t dax_lock_mapping_entry(struct address_space *mapping,
                unsigned long index, struct page **page)
{
        return 0;
}

static inline void dax_unlock_mapping_entry(struct address_space *mapping,
                unsigned long index, dax_entry_t cookie)
{
}
#endif

int dax_file_unshare(struct inode *inode, loff_t pos, loff_t len,
                const struct iomap_ops *ops);
int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
                const struct iomap_ops *ops);
int dax_truncate_page(struct inode *inode, loff_t pos, bool *did_zero,
                const struct iomap_ops *ops);

static inline bool dax_page_is_idle(struct page *page)
{
        return page && page_ref_count(page) == 0;
}

#if IS_ENABLED(CONFIG_DAX)
int dax_read_lock(void);
void dax_read_unlock(int id);
#else
static inline int dax_read_lock(void)
{
        return 0;
}

static inline void dax_read_unlock(int id)
{
}
#endif /* CONFIG_DAX */

#if !IS_ENABLED(CONFIG_FS_DAX)
static inline int __must_check dax_break_layout(struct inode *inode,
                            loff_t start, loff_t end, void (cb)(struct inode *))
{
        return 0;
}

static inline void dax_break_layout_final(struct inode *inode)
{
}
#endif

bool dax_alive(struct dax_device *dax_dev);
void *dax_get_private(struct dax_device *dax_dev);
int dax_set_ops(struct dax_device *dax_dev, const struct dax_operations *ops);
long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
                enum dax_access_mode mode, void **kaddr, unsigned long *pfn);
size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
                size_t bytes, struct iov_iter *i);
size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
                size_t bytes, struct iov_iter *i);
int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
                        size_t nr_pages);
int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, u64 len,
                int mf_flags);
void dax_flush(struct dax_device *dax_dev, void *addr, size_t size);

ssize_t dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
                const struct iomap_ops *ops);
vm_fault_t dax_iomap_fault(struct vm_fault *vmf, unsigned int order,
                        unsigned long *pfnp, int *errp,
                        const struct iomap_ops *ops);
vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf,
                unsigned int order, unsigned long pfn);
int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index);
void dax_delete_mapping_range(struct address_space *mapping,
                                loff_t start, loff_t end);
int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
                                      pgoff_t index);
int __must_check dax_break_layout(struct inode *inode, loff_t start,
                                loff_t end, void (cb)(struct inode *));
static inline int __must_check dax_break_layout_inode(struct inode *inode,
                                                void (cb)(struct inode *))
{
        return dax_break_layout(inode, 0, LLONG_MAX, cb);
}
void dax_break_layout_final(struct inode *inode);
int dax_dedupe_file_range_compare(struct inode *src, loff_t srcoff,
                                  struct inode *dest, loff_t destoff,
                                  loff_t len, bool *is_same,
                                  const struct iomap_ops *ops);
int dax_remap_file_range_prep(struct file *file_in, loff_t pos_in,
                              struct file *file_out, loff_t pos_out,
                              loff_t *len, unsigned int remap_flags,
                              const struct iomap_ops *ops);
static inline bool dax_mapping(struct address_space *mapping)
{
        return mapping->host && IS_DAX(mapping->host);
}

/*
 * Due to dax's memory and block duo personalities, hwpoison reporting
 * takes into consideration which personality is presently visible.
 * When dax acts like a block device, such as in block IO, an encounter of
 * dax hwpoison is reported as -EIO.
 * When dax acts like memory, such as in page fault, a detection of hwpoison
 * is reported as -EHWPOISON which leads to VM_FAULT_HWPOISON.
 */
static inline int dax_mem2blk_err(int err)
{
        return (err == -EHWPOISON) ? -EIO : err;
}

#ifdef CONFIG_DEV_DAX_HMEM_DEVICES
void hmem_register_resource(int target_nid, struct resource *r);
#else
static inline void hmem_register_resource(int target_nid, struct resource *r)
{
}
#endif

typedef int (*walk_hmem_fn)(struct device *dev, int target_nid,
                            const struct resource *res);
int walk_hmem_resources(struct device *dev, walk_hmem_fn fn);
#endif






























































































































































































































































































    2 




    3 
    2 
























































   50 




   43 







   44 
















   99 





  269 






  126 





   21 







    7 

    7 

    7 









   13 

   13 

   13 













































   43 











   10 



   10 




   12 



   11 



   11 











    3 














    4 

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
/* SPDX-License-Identifier: GPL-2.0-only */
/* Copyright (C) 2000-2002 Joakim Axelsson <gozem@linux.nu>
 *                         Patrick Schaaf <bof@bof.de>
 *                         Martin Josefsson <gandalf@wlug.westbo.se>
 * Copyright (C) 2003-2013 Jozsef Kadlecsik <kadlec@netfilter.org>
 */
#ifndef _IP_SET_H
#define _IP_SET_H

#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/netlink.h>
#include <linux/netfilter.h>
#include <linux/netfilter/x_tables.h>
#include <linux/stringify.h>
#include <linux/vmalloc.h>
#include <net/netlink.h>
#include <uapi/linux/netfilter/ipset/ip_set.h>

#define _IP_SET_MODULE_DESC(a, b, c)                \
        MODULE_DESCRIPTION(a " type of IP sets, revisions " b "-" c)
#define IP_SET_MODULE_DESC(a, b, c)                \
        _IP_SET_MODULE_DESC(a, __stringify(b), __stringify(c))

/* Set features */
enum ip_set_feature {
        IPSET_TYPE_IP_FLAG = 0,
        IPSET_TYPE_IP = (1 << IPSET_TYPE_IP_FLAG),
        IPSET_TYPE_PORT_FLAG = 1,
        IPSET_TYPE_PORT = (1 << IPSET_TYPE_PORT_FLAG),
        IPSET_TYPE_MAC_FLAG = 2,
        IPSET_TYPE_MAC = (1 << IPSET_TYPE_MAC_FLAG),
        IPSET_TYPE_IP2_FLAG = 3,
        IPSET_TYPE_IP2 = (1 << IPSET_TYPE_IP2_FLAG),
        IPSET_TYPE_NAME_FLAG = 4,
        IPSET_TYPE_NAME = (1 << IPSET_TYPE_NAME_FLAG),
        IPSET_TYPE_IFACE_FLAG = 5,
        IPSET_TYPE_IFACE = (1 << IPSET_TYPE_IFACE_FLAG),
        IPSET_TYPE_MARK_FLAG = 6,
        IPSET_TYPE_MARK = (1 << IPSET_TYPE_MARK_FLAG),
        IPSET_TYPE_NOMATCH_FLAG = 7,
        IPSET_TYPE_NOMATCH = (1 << IPSET_TYPE_NOMATCH_FLAG),
        /* Strictly speaking not a feature, but a flag for dumping:
         * this settype must be dumped last */
        IPSET_DUMP_LAST_FLAG = 8,
        IPSET_DUMP_LAST = (1 << IPSET_DUMP_LAST_FLAG),
};

/* Set extensions */
enum ip_set_extension {
        IPSET_EXT_BIT_TIMEOUT = 0,
        IPSET_EXT_TIMEOUT = (1 << IPSET_EXT_BIT_TIMEOUT),
        IPSET_EXT_BIT_COUNTER = 1,
        IPSET_EXT_COUNTER = (1 << IPSET_EXT_BIT_COUNTER),
        IPSET_EXT_BIT_COMMENT = 2,
        IPSET_EXT_COMMENT = (1 << IPSET_EXT_BIT_COMMENT),
        IPSET_EXT_BIT_SKBINFO = 3,
        IPSET_EXT_SKBINFO = (1 << IPSET_EXT_BIT_SKBINFO),
        /* Mark set with an extension which needs to call destroy */
        IPSET_EXT_BIT_DESTROY = 7,
        IPSET_EXT_DESTROY = (1 << IPSET_EXT_BIT_DESTROY),
};

#define SET_WITH_TIMEOUT(s)        ((s)->extensions & IPSET_EXT_TIMEOUT)
#define SET_WITH_COUNTER(s)        ((s)->extensions & IPSET_EXT_COUNTER)
#define SET_WITH_COMMENT(s)        ((s)->extensions & IPSET_EXT_COMMENT)
#define SET_WITH_SKBINFO(s)        ((s)->extensions & IPSET_EXT_SKBINFO)
#define SET_WITH_FORCEADD(s)        ((s)->flags & IPSET_CREATE_FLAG_FORCEADD)

/* Extension id, in size order */
enum ip_set_ext_id {
        IPSET_EXT_ID_COUNTER = 0,
        IPSET_EXT_ID_TIMEOUT,
        IPSET_EXT_ID_SKBINFO,
        IPSET_EXT_ID_COMMENT,
        IPSET_EXT_ID_MAX,
};

struct ip_set;

/* Extension type */
struct ip_set_ext_type {
        /* Destroy extension private data (can be NULL) */
        void (*destroy)(struct ip_set *set, void *ext);
        enum ip_set_extension type;
        enum ipset_cadt_flags flag;
        /* Size and minimal alignment */
        u8 len;
        u8 align;
};

extern const struct ip_set_ext_type ip_set_extensions[];

struct ip_set_counter {
        atomic64_t bytes;
        atomic64_t packets;
};

struct ip_set_comment_rcu {
        struct rcu_head rcu;
        char str[];
};

struct ip_set_comment {
        struct ip_set_comment_rcu __rcu *c;
};

struct ip_set_skbinfo {
        u32 skbmark;
        u32 skbmarkmask;
        u32 skbprio;
        u16 skbqueue;
        u16 __pad;
};

struct ip_set_ext {
        struct ip_set_skbinfo skbinfo;
        u64 packets;
        u64 bytes;
        char *comment;
        u32 timeout;
        u8 packets_op;
        u8 bytes_op;
        bool target;
};

#define ext_timeout(e, s)        \
((unsigned long *)(((void *)(e)) + (s)->offset[IPSET_EXT_ID_TIMEOUT]))
#define ext_counter(e, s)        \
((struct ip_set_counter *)(((void *)(e)) + (s)->offset[IPSET_EXT_ID_COUNTER]))
#define ext_comment(e, s)        \
((struct ip_set_comment *)(((void *)(e)) + (s)->offset[IPSET_EXT_ID_COMMENT]))
#define ext_skbinfo(e, s)        \
((struct ip_set_skbinfo *)(((void *)(e)) + (s)->offset[IPSET_EXT_ID_SKBINFO]))

typedef int (*ipset_adtfn)(struct ip_set *set, void *value,
                           const struct ip_set_ext *ext,
                           struct ip_set_ext *mext, u32 cmdflags);

/* Kernel API function options */
struct ip_set_adt_opt {
        u8 family;                /* Actual protocol family */
        u8 dim;                        /* Dimension of match/target */
        u8 flags;                /* Direction and negation flags */
        u32 cmdflags;                /* Command-like flags */
        struct ip_set_ext ext;        /* Extensions */
};

/* Set type, variant-specific part */
struct ip_set_type_variant {
        /* Kernelspace: test/add/del entries
         *                returns negative error code,
         *                        zero for no match/success to add/delete
         *                        positive for matching element */
        int (*kadt)(struct ip_set *set, const struct sk_buff *skb,
                    const struct xt_action_param *par,
                    enum ipset_adt adt, struct ip_set_adt_opt *opt);

        /* Userspace: test/add/del entries
         *                returns negative error code,
         *                        zero for no match/success to add/delete
         *                        positive for matching element */
        int (*uadt)(struct ip_set *set, struct nlattr *tb[],
                    enum ipset_adt adt, u32 *lineno, u32 flags, bool retried);

        /* Low level add/del/test functions */
        ipset_adtfn adt[IPSET_ADT_MAX];

        /* When adding entries and set is full, try to resize the set */
        int (*resize)(struct ip_set *set, bool retried);
        /* Destroy the set */
        void (*destroy)(struct ip_set *set);
        /* Flush the elements */
        void (*flush)(struct ip_set *set);
        /* Expire entries before listing */
        void (*expire)(struct ip_set *set);
        /* List set header data */
        int (*head)(struct ip_set *set, struct sk_buff *skb);
        /* List elements */
        int (*list)(const struct ip_set *set, struct sk_buff *skb,
                    struct netlink_callback *cb);
        /* Keep listing private when resizing runs parallel */
        void (*uref)(struct ip_set *set, struct netlink_callback *cb,
                     bool start);

        /* Return true if "b" set is the same as "a"
         * according to the create set parameters */
        bool (*same_set)(const struct ip_set *a, const struct ip_set *b);
        /* Cancel ongoing garbage collectors before destroying the set*/
        void (*cancel_gc)(struct ip_set *set);
        /* Region-locking is used */
        bool region_lock;
};

struct ip_set_region {
        spinlock_t lock;        /* Region lock */
        size_t ext_size;        /* Size of the dynamic extensions */
        u32 elements;                /* Number of elements vs timeout */
};

/* Max range where every element is added/deleted in one step */
#define IPSET_MAX_RANGE                (1<<14)

/* The max revision number supported by any set type + 1 */
#define IPSET_REVISION_MAX        9

/* The core set type structure */
struct ip_set_type {
        struct list_head list;

        /* Typename */
        char name[IPSET_MAXNAMELEN];
        /* Protocol version */
        u8 protocol;
        /* Set type dimension */
        u8 dimension;
        /*
         * Supported family: may be NFPROTO_UNSPEC for both
         * NFPROTO_IPV4/NFPROTO_IPV6.
         */
        u8 family;
        /* Type revisions */
        u8 revision_min, revision_max;
        /* Revision-specific supported (create) flags */
        u8 create_flags[IPSET_REVISION_MAX+1];
        /* Set features to control swapping */
        u16 features;

        /* Create set */
        int (*create)(struct net *net, struct ip_set *set,
                      struct nlattr *tb[], u32 flags);

        /* Attribute policies */
        const struct nla_policy create_policy[IPSET_ATTR_CREATE_MAX + 1];
        const struct nla_policy adt_policy[IPSET_ATTR_ADT_MAX + 1];

        /* Set this to THIS_MODULE if you are a module, otherwise NULL */
        struct module *me;
};

/* register and unregister set type */
extern int ip_set_type_register(struct ip_set_type *set_type);
extern void ip_set_type_unregister(struct ip_set_type *set_type);

/* A generic IP set */
struct ip_set {
        /* for set destruction */
        struct rcu_work rwork;
        /* The name of the set */
        char name[IPSET_MAXNAMELEN];
        /* Lock protecting the set data */
        spinlock_t lock;
        /* References to the set */
        u32 ref;
        /* References to the set for netlink events like dump,
         * ref can be swapped out by ip_set_swap
         */
        u32 ref_netlink;
        /* The core set type */
        struct ip_set_type *type;
        /* The type variant doing the real job */
        const struct ip_set_type_variant *variant;
        /* The actual INET family of the set */
        u8 family;
        /* The type revision */
        u8 revision;
        /* Extensions */
        u8 extensions;
        /* Create flags */
        u8 flags;
        /* Default timeout value, if enabled */
        u32 timeout;
        /* Number of elements (vs timeout) */
        u32 elements;
        /* Size of the dynamic extensions (vs timeout) */
        atomic64_t ext_size;
        /* Element data size */
        size_t dsize;
        /* Offsets to extensions in elements */
        size_t offset[IPSET_EXT_ID_MAX];
        /* The type specific data */
        void *data;
};

static inline void
ip_set_ext_destroy(struct ip_set *set, void *data)
{
        /* Check that the extension is enabled for the set and
         * call it's destroy function for its extension part in data.
         */
        if (SET_WITH_COMMENT(set)) {
                struct ip_set_comment *c = ext_comment(data, set);

                ip_set_extensions[IPSET_EXT_ID_COMMENT].destroy(set, c);
        }
}

int ip_set_put_flags(struct sk_buff *skb, struct ip_set *set);

/* Netlink CB args */
enum {
        IPSET_CB_NET = 0,        /* net namespace */
        IPSET_CB_PROTO,                /* ipset protocol */
        IPSET_CB_DUMP,                /* dump single set/all sets */
        IPSET_CB_INDEX,                /* set index */
        IPSET_CB_PRIVATE,        /* set private data */
        IPSET_CB_ARG0,                /* type specific */
};

/* register and unregister set references */
extern ip_set_id_t ip_set_get_byname(struct net *net,
                                     const struct nlattr *name, struct ip_set **set);
extern void ip_set_put_byindex(struct net *net, ip_set_id_t index);
extern void ip_set_name_byindex(struct net *net, ip_set_id_t index, char *name);
extern ip_set_id_t ip_set_nfnl_get_byindex(struct net *net, ip_set_id_t index);
extern void ip_set_nfnl_put(struct net *net, ip_set_id_t index);

/* API for iptables set match, and SET target */

extern int ip_set_add(ip_set_id_t id, const struct sk_buff *skb,
                      const struct xt_action_param *par,
                      struct ip_set_adt_opt *opt);
extern int ip_set_del(ip_set_id_t id, const struct sk_buff *skb,
                      const struct xt_action_param *par,
                      struct ip_set_adt_opt *opt);
extern int ip_set_test(ip_set_id_t id, const struct sk_buff *skb,
                       const struct xt_action_param *par,
                       struct ip_set_adt_opt *opt);

/* Utility functions */
extern void *ip_set_alloc(size_t size);
extern void ip_set_free(void *members);
extern int ip_set_get_ipaddr4(struct nlattr *nla,  __be32 *ipaddr);
extern int ip_set_get_ipaddr6(struct nlattr *nla, union nf_inet_addr *ipaddr);
extern size_t ip_set_elem_len(struct ip_set *set, struct nlattr *tb[],
                              size_t len, size_t align);
extern int ip_set_get_extensions(struct ip_set *set, struct nlattr *tb[],
                                 struct ip_set_ext *ext);
extern int ip_set_put_extensions(struct sk_buff *skb, const struct ip_set *set,
                                 const void *e, bool active);
extern bool ip_set_match_extensions(struct ip_set *set,
                                    const struct ip_set_ext *ext,
                                    struct ip_set_ext *mext,
                                    u32 flags, void *data);

static inline int
ip_set_get_hostipaddr4(struct nlattr *nla, u32 *ipaddr)
{
        __be32 ip;
        int ret = ip_set_get_ipaddr4(nla, &ip);

        if (ret)
                return ret;
        *ipaddr = ntohl(ip);
        return 0;
}

/* Ignore IPSET_ERR_EXIST errors if asked to do so? */
static inline bool
ip_set_eexist(int ret, u32 flags)
{
        return ret == -IPSET_ERR_EXIST && (flags & IPSET_FLAG_EXIST);
}

/* Match elements marked with nomatch */
static inline bool
ip_set_enomatch(int ret, u32 flags, enum ipset_adt adt, struct ip_set *set)
{
        return adt == IPSET_TEST &&
               (set->type->features & IPSET_TYPE_NOMATCH) &&
               ((flags >> 16) & IPSET_FLAG_NOMATCH) &&
               (ret > 0 || ret == -ENOTEMPTY);
}

/* Check the NLA_F_NET_BYTEORDER flag */
static inline bool
ip_set_attr_netorder(struct nlattr *tb[], int type)
{
        return tb[type] && (tb[type]->nla_type & NLA_F_NET_BYTEORDER);
}

static inline bool
ip_set_optattr_netorder(struct nlattr *tb[], int type)
{
        return !tb[type] || (tb[type]->nla_type & NLA_F_NET_BYTEORDER);
}

/* Useful converters */
static inline u32
ip_set_get_h32(const struct nlattr *attr)
{
        return ntohl(nla_get_be32(attr));
}

static inline u16
ip_set_get_h16(const struct nlattr *attr)
{
        return ntohs(nla_get_be16(attr));
}

static inline int nla_put_ipaddr4(struct sk_buff *skb, int type, __be32 ipaddr)
{
        struct nlattr *__nested = nla_nest_start(skb, type);
        int ret;

        if (!__nested)
                return -EMSGSIZE;
        ret = nla_put_in_addr(skb, IPSET_ATTR_IPADDR_IPV4, ipaddr);
        if (!ret)
                nla_nest_end(skb, __nested);
        return ret;
}

static inline int nla_put_ipaddr6(struct sk_buff *skb, int type,
                                  const struct in6_addr *ipaddrptr)
{
        struct nlattr *__nested = nla_nest_start(skb, type);
        int ret;

        if (!__nested)
                return -EMSGSIZE;
        ret = nla_put_in6_addr(skb, IPSET_ATTR_IPADDR_IPV6, ipaddrptr);
        if (!ret)
                nla_nest_end(skb, __nested);
        return ret;
}

/* Get address from skbuff */
static inline __be32
ip4addr(const struct sk_buff *skb, bool src)
{
        return src ? ip_hdr(skb)->saddr : ip_hdr(skb)->daddr;
}

static inline void
ip4addrptr(const struct sk_buff *skb, bool src, __be32 *addr)
{
        *addr = src ? ip_hdr(skb)->saddr : ip_hdr(skb)->daddr;
}

static inline void
ip6addrptr(const struct sk_buff *skb, bool src, struct in6_addr *addr)
{
        memcpy(addr, src ? &ipv6_hdr(skb)->saddr : &ipv6_hdr(skb)->daddr,
               sizeof(*addr));
}

/* How often should the gc be run by default */
#define IPSET_GC_TIME                        (3 * 60)

/* Timeout period depending on the timeout value of the given set */
#define IPSET_GC_PERIOD(timeout) \
        ((timeout/3) ? min_t(u32, (timeout)/3, IPSET_GC_TIME) : 1)

/* Entry is set with no timeout value */
#define IPSET_ELEM_PERMANENT        0

/* Set is defined with timeout support: timeout value may be 0 */
#define IPSET_NO_TIMEOUT        UINT_MAX

/* Max timeout value, see msecs_to_jiffies() in jiffies.h */
#define IPSET_MAX_TIMEOUT        (UINT_MAX >> 1)/MSEC_PER_SEC

#define ip_set_adt_opt_timeout(opt, set)        \
((opt)->ext.timeout != IPSET_NO_TIMEOUT ? (opt)->ext.timeout : (set)->timeout)

static inline unsigned int
ip_set_timeout_uget(struct nlattr *tb)
{
        unsigned int timeout = ip_set_get_h32(tb);

        /* Normalize to fit into jiffies */
        if (timeout > IPSET_MAX_TIMEOUT)
                timeout = IPSET_MAX_TIMEOUT;

        return timeout;
}

static inline bool
ip_set_timeout_expired(const unsigned long *t)
{
        return *t != IPSET_ELEM_PERMANENT && time_is_before_jiffies(*t);
}

static inline void
ip_set_timeout_set(unsigned long *timeout, u32 value)
{
        unsigned long t;

        if (!value) {
                *timeout = IPSET_ELEM_PERMANENT;
                return;
        }

        t = msecs_to_jiffies(value * MSEC_PER_SEC) + jiffies;
        if (t == IPSET_ELEM_PERMANENT)
                /* Bingo! :-) */
                t--;
        *timeout = t;
}

void ip_set_init_comment(struct ip_set *set, struct ip_set_comment *comment,
                         const struct ip_set_ext *ext);

static inline void
ip_set_init_counter(struct ip_set_counter *counter,
                    const struct ip_set_ext *ext)
{
        if (ext->bytes != ULLONG_MAX)
                atomic64_set(&(counter)->bytes, (long long)(ext->bytes));
        if (ext->packets != ULLONG_MAX)
                atomic64_set(&(counter)->packets, (long long)(ext->packets));
}

static inline void
ip_set_init_skbinfo(struct ip_set_skbinfo *skbinfo,
                    const struct ip_set_ext *ext)
{
        *skbinfo = ext->skbinfo;
}

static inline void
nf_inet_addr_mask_inplace(union nf_inet_addr *a1,
                          const union nf_inet_addr *mask)
{
        a1->all[0] &= mask->all[0];
        a1->all[1] &= mask->all[1];
        a1->all[2] &= mask->all[2];
        a1->all[3] &= mask->all[3];
}

#define IP_SET_INIT_KEXT(skb, opt, set)                        \
        { .bytes = (skb)->len, .packets = 1, .target = true,\
          .timeout = ip_set_adt_opt_timeout(opt, set) }

#define IP_SET_INIT_UEXT(set)                                \
        { .bytes = ULLONG_MAX, .packets = ULLONG_MAX,        \
          .timeout = (set)->timeout }

#define IPSET_CONCAT(a, b)                a##b
#define IPSET_TOKEN(a, b)                IPSET_CONCAT(a, b)

#endif /*_IP_SET_H */



















   16 


   16 






   16 






    2 










    2 




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
/*
 * JFFS2 -- Journalling Flash File System, Version 2.
 *
 * Copyright © 2001-2007 Red Hat, Inc.
 *
 * Created by David Woodhouse <dwmw2@infradead.org>
 *
 * For licensing information, see the file 'LICENCE' in this directory.
 *
 */

#include <linux/kernel.h>
#include <linux/mtd/mtd.h>
#include "nodelist.h"

int jffs2_flash_direct_writev(struct jffs2_sb_info *c, const struct kvec *vecs,
                              unsigned long count, loff_t to, size_t *retlen)
{
        if (!jffs2_is_writebuffered(c)) {
                if (jffs2_sum_active()) {
                        int res;
                        res = jffs2_sum_add_kvec(c, vecs, count, (uint32_t) to);
                        if (res) {
                                return res;
                        }
                }
        }

        return mtd_writev(c->mtd, vecs, count, to, retlen);
}

int jffs2_flash_direct_write(struct jffs2_sb_info *c, loff_t ofs, size_t len,
                        size_t *retlen, const u_char *buf)
{
        int ret;
        ret = mtd_write(c->mtd, ofs, len, retlen, buf);

        if (jffs2_sum_active()) {
                struct kvec vecs[1];
                int res;

                vecs[0].iov_base = (unsigned char *) buf;
                vecs[0].iov_len = len;

                res = jffs2_sum_add_kvec(c, vecs, 1, (uint32_t) ofs);
                if (res) {
                        return res;
                }
        }
        return ret;
}






























































   85 


   85 


   85 
























































  115 


  114 
  114 

  114 




  115 
  115 







  115 





   34 














   98 


    1 




   98 
   98 




   97 





   97 






   97 
    2 










   97 































  110 


   91 

   73 

  110 
  110 





  110 
    2 





  109 






  109 

    2 





  107 
  107 















































   98 




   98 






   98 
   98 






   98 
























   18 






   18 










    1 





   17 
    1 




   17 




   18 
   17 




    2 








  109 






  108 

    4 




  105 
  105 


  109 

































   34 




















   34 


   34 






   34 





   34 












   34 








   36 


   36 











   36 


   36 





    2 
    2 




   34 



   34 





   34 









   34 


















































































































































































































   36 

   41 


    5 
    1 





   36 







   36 





   36 







   34 



    2 


   34 








   34 






   34 

   34 













   34 




   34 
    2 


   34 
   36 

   40 
   36 

   36 

   37 
    3 
   38 










  115 











  115 



    1 


    1 



  114 








  114 
    1 





  114 




   70 
   56 





   41 






   41 







   34 







  110 


  114 

  111 
  115 





  188 















  188 

















  101 

    1 

    4 







   83 




   93 







    4 



    4 

    4 
    4 


    4 














    4 





    1 





    3 







   93 








   93 







   93 











    6 
    3 



    3 


    3 


    3 






    6 
    6 



    3 
    3 
   93 















   83 









   83 










   79 










   83 



   83 










   81 









   81 


    2 
    1 



    1 


    1 


    1 






    2 
    2 



    1 
    1 
   83 









   79 






   57 
    1 

   79 













  112 






  112 




  104 
   99 


    7 
    5 






   48 



    6 









    6 


    6 









   43 

    1 






   12 
   12 



    6 
    6 
  112 






  104 




























  103 






  103 



   47 


   25 


   40 











   64 
   64 



   64 





   64 












  103 






  103 

  103 

  103 




  103 

   62 





  103 


  103 
  103 



  103 






  103 

  103 







  103 












  103 

   63 

  103 



  103 
  103 



  103 


  103 








  103 








  103 





  103 

  103 













   88 


  103 














  103 


  103 


  103 





















































































   63 





   63 
   31 



   63 
   63 











   48 






   63 





   63 



    2 



    2 







   63 














   90 

   90 
   90 




   88 

   88 
   75 




   75 




   90 










  103 









  103 





  103 









































   88 




  103 
   63 





  103 
   77 



















   18 








   17 




   17 



    1 




   17 
   17 





   17 






   16 





   16 










   17 


   17 















  103 











  103 



    8 


   99 





   99 
   10 


    3 
   10 



   99 




   12 
















   99 
   12 




   99 


   99 


   99 
   99 





   99 














   99 











   99 


   99 







   99 














   99 




   99 

   99 
   99 

  103 
    8 
  103 










  109 


  107 




  109 
  109 

  109 



    1 

  109 

    4 














   18 



    1 


   17 

    1 






  103 










  103 
  103 





   99 
   38 

   85 


   20 
    8 



   12 





   12 
   12 
   12 

    5 


















   12 
   12 
   10 





   12 



  103 
   44 

   17 



    4 
   12 
  109 











   33 


   33 
   33 

   33 





    1 


   32 










   33 
























   14 
   14 
   74 
    4 




    4 



















    2 
    2 
    2 
    2 

    2 






    2 





    2 












    2 







    2 

    2 

    2 
    2 

    2 


    2 











    2 












    2 






    2 










    2 














    2 


    2 



    2 





    2 

    2 











   78 


   78 
   78 
   78 

   78 

   78 





    4 


   74 



   74 








   78 













   63 
    4 
   63 







    7 




    7 

    7 
    7 











    2 



    2 





    2 


















   75 




   75 



   75 
   56 

   56 






   56 

   72 









   72 








   63 

   63 

   63 




   19 
   19 
    9 







    9 
   75 








   56 























   10 

   10 

   10 
   10 





   10 



   10 







   10 
   10 



    2 

   10 














   10 
    2 



   10 





    2 

   10 


















































































































































































































   10 












   10 

   10 




   10 





   10 



   10 
    2 
   10 







   10 









   10 












    8 
    1 






   10 
   10 









    1 








    7 
    7 




    7 




















    2 




    2 






    2 







    2 



    2 








    1 














    1 






















   35 



   35 



   35 



















   35 
   35 
    6 







   35 


   33 

    3 
    3 





    3 









   35 











    1 





    1 

    1 





























    1 

    1 
    1 
    1 













































































    1 



    1 


    1 












































    1 
    1 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * suballoc.c
 *
 * metadata alloc and free
 * Inspired by ext3 block groups.
 *
 * Copyright (C) 2002, 2004 Oracle.  All rights reserved.
 */

#include <linux/fs.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/highmem.h>

#include <cluster/masklog.h>

#include "ocfs2.h"

#include "alloc.h"
#include "blockcheck.h"
#include "dlmglue.h"
#include "inode.h"
#include "journal.h"
#include "localalloc.h"
#include "suballoc.h"
#include "super.h"
#include "sysfile.h"
#include "uptodate.h"
#include "ocfs2_trace.h"

#include "buffer_head_io.h"

#define NOT_ALLOC_NEW_GROUP                0
#define ALLOC_NEW_GROUP                        0x1
#define ALLOC_GROUPS_FROM_GLOBAL        0x2

#define OCFS2_MAX_TO_STEAL                1024

struct ocfs2_suballoc_result {
        u64                sr_bg_blkno;        /* The bg we allocated from.  Set
                                           to 0 when a block group is
                                           contiguous. */
        u64                sr_bg_stable_blkno; /*
                                             * Doesn't change, always
                                             * set to target block
                                             * group descriptor
                                             * block.
                                             */
        u64                sr_blkno;        /* The first allocated block */
        unsigned int        sr_bit_offset;        /* The bit in the bg */
        unsigned int        sr_bits;        /* How many bits we claimed */
        unsigned int        sr_max_contig_bits; /* The length for contiguous
                                             * free bits, only available
                                             * for cluster group
                                             */
};

static u64 ocfs2_group_from_res(struct ocfs2_suballoc_result *res)
{
        if (res->sr_blkno == 0)
                return 0;

        if (res->sr_bg_blkno)
                return res->sr_bg_blkno;

        return ocfs2_which_suballoc_group(res->sr_blkno, res->sr_bit_offset);
}

static inline u16 ocfs2_find_victim_chain(struct ocfs2_chain_list *cl);
static int ocfs2_block_group_fill(handle_t *handle,
                                  struct inode *alloc_inode,
                                  struct buffer_head *bg_bh,
                                  u64 group_blkno,
                                  unsigned int group_clusters,
                                  u16 my_chain,
                                  struct ocfs2_chain_list *cl);
static int ocfs2_block_group_alloc(struct ocfs2_super *osb,
                                   struct inode *alloc_inode,
                                   struct buffer_head *bh,
                                   u64 max_block,
                                   u64 *last_alloc_group,
                                   int flags);

static int ocfs2_cluster_group_search(struct inode *inode,
                                      struct buffer_head *group_bh,
                                      u32 bits_wanted, u32 min_bits,
                                      u64 max_block,
                                      struct ocfs2_suballoc_result *res);
static int ocfs2_block_group_search(struct inode *inode,
                                    struct buffer_head *group_bh,
                                    u32 bits_wanted, u32 min_bits,
                                    u64 max_block,
                                    struct ocfs2_suballoc_result *res);
static int ocfs2_claim_suballoc_bits(struct ocfs2_alloc_context *ac,
                                     handle_t *handle,
                                     u32 bits_wanted,
                                     u32 min_bits,
                                     struct ocfs2_suballoc_result *res);
static int ocfs2_test_bg_bit_allocatable(struct buffer_head *bg_bh,
                                         int nr);
static int ocfs2_relink_block_group(handle_t *handle,
                                    struct inode *alloc_inode,
                                    struct buffer_head *fe_bh,
                                    struct buffer_head *bg_bh,
                                    struct buffer_head *prev_bg_bh,
                                    u16 chain);
static inline int ocfs2_block_group_reasonably_empty(struct ocfs2_group_desc *bg,
                                                     u32 wanted);
static inline u32 ocfs2_desc_bitmap_to_cluster_off(struct inode *inode,
                                                   u64 bg_blkno,
                                                   u16 bg_bit_off);
static inline void ocfs2_block_to_cluster_group(struct inode *inode,
                                                u64 data_blkno,
                                                u64 *bg_blkno,
                                                u16 *bg_bit_off);
static int ocfs2_reserve_clusters_with_limit(struct ocfs2_super *osb,
                                             u32 bits_wanted, u64 max_block,
                                             int flags,
                                             struct ocfs2_alloc_context **ac);

void ocfs2_free_ac_resource(struct ocfs2_alloc_context *ac)
{
        struct inode *inode = ac->ac_inode;

        if (inode) {
                if (ac->ac_which != OCFS2_AC_USE_LOCAL)
                        ocfs2_inode_unlock(inode, 1);

                inode_unlock(inode);

                iput(inode);
                ac->ac_inode = NULL;
        }
        brelse(ac->ac_bh);
        ac->ac_bh = NULL;
        ac->ac_resv = NULL;
        kfree(ac->ac_find_loc_priv);
        ac->ac_find_loc_priv = NULL;
}

void ocfs2_free_alloc_context(struct ocfs2_alloc_context *ac)
{
        ocfs2_free_ac_resource(ac);
        kfree(ac);
}

static u32 ocfs2_bits_per_group(struct ocfs2_chain_list *cl)
{
        return (u32)le16_to_cpu(cl->cl_cpg) * (u32)le16_to_cpu(cl->cl_bpc);
}

#define do_error(fmt, ...)                                                \
do {                                                                        \
        if (resize)                                                        \
                mlog(ML_ERROR, fmt, ##__VA_ARGS__);                        \
        else                                                                \
                return ocfs2_error(sb, fmt, ##__VA_ARGS__);                \
} while (0)

static int ocfs2_validate_gd_self(struct super_block *sb,
                                  struct buffer_head *bh,
                                  int resize)
{
        struct ocfs2_group_desc *gd = (struct ocfs2_group_desc *)bh->b_data;

        if (!OCFS2_IS_VALID_GROUP_DESC(gd)) {
                do_error("Group descriptor #%llu has bad signature %.*s\n",
                         (unsigned long long)bh->b_blocknr, 7,
                         gd->bg_signature);
        }

        if (le64_to_cpu(gd->bg_blkno) != bh->b_blocknr) {
                do_error("Group descriptor #%llu has an invalid bg_blkno of %llu\n",
                         (unsigned long long)bh->b_blocknr,
                         (unsigned long long)le64_to_cpu(gd->bg_blkno));
        }

        if (le32_to_cpu(gd->bg_generation) != OCFS2_SB(sb)->fs_generation) {
                do_error("Group descriptor #%llu has an invalid fs_generation of #%u\n",
                         (unsigned long long)bh->b_blocknr,
                         le32_to_cpu(gd->bg_generation));
        }

        if (le16_to_cpu(gd->bg_free_bits_count) > le16_to_cpu(gd->bg_bits)) {
                do_error("Group descriptor #%llu has bit count %u but claims that %u are free\n",
                         (unsigned long long)bh->b_blocknr,
                         le16_to_cpu(gd->bg_bits),
                         le16_to_cpu(gd->bg_free_bits_count));
        }

        if (le16_to_cpu(gd->bg_bits) > (8 * le16_to_cpu(gd->bg_size))) {
                do_error("Group descriptor #%llu has bit count %u but max bitmap bits of %u\n",
                         (unsigned long long)bh->b_blocknr,
                         le16_to_cpu(gd->bg_bits),
                         8 * le16_to_cpu(gd->bg_size));
        }

        /*
         * For discontiguous block groups, validate the on-disk extent list
         * against the maximum number of extent records that can physically
         * fit in a single block.
         */
        if (ocfs2_gd_is_discontig(gd)) {
                u16 max_recs = ocfs2_extent_recs_per_gd(sb);
                u16 l_count = le16_to_cpu(gd->bg_list.l_count);
                u16 l_next_free_rec = le16_to_cpu(gd->bg_list.l_next_free_rec);

                if (l_count != max_recs) {
                        do_error("Group descriptor #%llu bad discontig l_count %u expected %u\n",
                                 (unsigned long long)bh->b_blocknr,
                                 l_count,
                                 max_recs);
                }

                if (l_next_free_rec > l_count) {
                        do_error("Group descriptor #%llu bad discontig l_next_free_rec %u max %u\n",
                                 (unsigned long long)bh->b_blocknr,
                                 l_next_free_rec,
                                 l_count);
                }
        }

        return 0;
}

static int ocfs2_validate_gd_parent(struct super_block *sb,
                                    struct ocfs2_dinode *di,
                                    struct buffer_head *bh,
                                    int resize)
{
        unsigned int max_bits;
        unsigned int max_bitmap_bits;
        unsigned int max_bitmap_size;
        int suballocator;
        struct ocfs2_group_desc *gd = (struct ocfs2_group_desc *)bh->b_data;

        suballocator = le64_to_cpu(di->i_blkno) != OCFS2_SB(sb)->bitmap_blkno;
        max_bitmap_size = ocfs2_group_bitmap_size(sb, suballocator,
                                                  OCFS2_SB(sb)->s_feature_incompat);
        max_bitmap_bits = max_bitmap_size * 8;

        if (di->i_blkno != gd->bg_parent_dinode) {
                do_error("Group descriptor #%llu has bad parent pointer (%llu, expected %llu)\n",
                         (unsigned long long)bh->b_blocknr,
                         (unsigned long long)le64_to_cpu(gd->bg_parent_dinode),
                         (unsigned long long)le64_to_cpu(di->i_blkno));
        }

        if (le16_to_cpu(gd->bg_size) > max_bitmap_size) {
                do_error("Group descriptor #%llu has bitmap size %u but physical max of %u\n",
                         (unsigned long long)bh->b_blocknr,
                         le16_to_cpu(gd->bg_size),
                         max_bitmap_size);
        }

        if (le16_to_cpu(gd->bg_bits) > max_bitmap_bits) {
                do_error("Group descriptor #%llu has bit count %u but physical max of %u\n",
                         (unsigned long long)bh->b_blocknr,
                         le16_to_cpu(gd->bg_bits),
                         max_bitmap_bits);
        }

        max_bits = le16_to_cpu(di->id2.i_chain.cl_cpg) * le16_to_cpu(di->id2.i_chain.cl_bpc);
        if (le16_to_cpu(gd->bg_bits) > max_bits) {
                do_error("Group descriptor #%llu has bit count of %u\n",
                         (unsigned long long)bh->b_blocknr,
                         le16_to_cpu(gd->bg_bits));
        }

        /* In resize, we may meet the case bg_chain == cl_next_free_rec. */
        if ((le16_to_cpu(gd->bg_chain) >
             le16_to_cpu(di->id2.i_chain.cl_next_free_rec)) ||
            ((le16_to_cpu(gd->bg_chain) ==
             le16_to_cpu(di->id2.i_chain.cl_next_free_rec)) && !resize)) {
                do_error("Group descriptor #%llu has bad chain %u\n",
                         (unsigned long long)bh->b_blocknr,
                         le16_to_cpu(gd->bg_chain));
        }

        return 0;
}

#undef do_error

/*
 * This version only prints errors.  It does not fail the filesystem, and
 * exists only for resize.
 */
int ocfs2_check_group_descriptor(struct super_block *sb,
                                 struct ocfs2_dinode *di,
                                 struct buffer_head *bh)
{
        int rc;
        struct ocfs2_group_desc *gd = (struct ocfs2_group_desc *)bh->b_data;

        BUG_ON(!buffer_uptodate(bh));

        /*
         * If the ecc fails, we return the error but otherwise
         * leave the filesystem running.  We know any error is
         * local to this block.
         */
        rc = ocfs2_validate_meta_ecc(sb, bh->b_data, &gd->bg_check);
        if (rc) {
                mlog(ML_ERROR,
                     "Checksum failed for group descriptor %llu\n",
                     (unsigned long long)bh->b_blocknr);
        } else
                rc = ocfs2_validate_gd_self(sb, bh, 1);
        if (!rc)
                rc = ocfs2_validate_gd_parent(sb, di, bh, 1);

        return rc;
}

static int ocfs2_validate_group_descriptor(struct super_block *sb,
                                           struct buffer_head *bh)
{
        int rc;
        struct ocfs2_group_desc *gd = (struct ocfs2_group_desc *)bh->b_data;

        trace_ocfs2_validate_group_descriptor(
                                        (unsigned long long)bh->b_blocknr);

        BUG_ON(!buffer_uptodate(bh));

        /*
         * If the ecc fails, we return the error but otherwise
         * leave the filesystem running.  We know any error is
         * local to this block.
         */
        rc = ocfs2_validate_meta_ecc(sb, bh->b_data, &gd->bg_check);
        if (rc)
                return rc;

        /*
         * Errors after here are fatal.
         */

        return ocfs2_validate_gd_self(sb, bh, 0);
}

/*
 * The hint group descriptor (gd) may already have been released
 * in _ocfs2_free_suballoc_bits(). We first check the gd signature,
 * then perform the standard ocfs2_read_group_descriptor() jobs.
 *
 * If the gd signature is invalid, we return 'rc=0' and set
 * '*released=1'. The caller is expected to handle this specific case.
 * Otherwise, we return the actual error code.
 *
 * We treat gd signature corruption case as a release case. The
 * caller ocfs2_claim_suballoc_bits() will use ocfs2_search_chain()
 * to search each gd block. The code will eventually find this
 * corrupted gd block - Late, but not missed.
 *
 * Note:
 * The caller is responsible for initializing the '*released' status.
 */
static int ocfs2_read_hint_group_descriptor(struct inode *inode,
                        struct ocfs2_dinode *di, u64 gd_blkno,
                        struct buffer_head **bh, int *released)
{
        int rc;
        struct buffer_head *tmp = *bh;
        struct ocfs2_group_desc *gd;

        rc = ocfs2_read_block(INODE_CACHE(inode), gd_blkno, &tmp, NULL);
        if (rc)
                goto out;

        gd = (struct ocfs2_group_desc *) tmp->b_data;
        if (!OCFS2_IS_VALID_GROUP_DESC(gd)) {
                /*
                 * Invalid gd cache was set in ocfs2_read_block(),
                 * which will affect block_group allocation.
                 * Path:
                 * ocfs2_reserve_suballoc_bits
                 *  ocfs2_block_group_alloc
                 *   ocfs2_block_group_alloc_contig
                 *    ocfs2_set_new_buffer_uptodate
                 */
                ocfs2_remove_from_cache(INODE_CACHE(inode), tmp);
                *released = 1; /* we return 'rc=0' for this case */
                goto free_bh;
        }

        /* below jobs same with ocfs2_read_group_descriptor() */
        if (!buffer_jbd(tmp)) {
                rc = ocfs2_validate_group_descriptor(inode->i_sb, tmp);
                if (rc)
                        goto free_bh;
        }

        rc = ocfs2_validate_gd_parent(inode->i_sb, di, tmp, 0);
        if (rc)
                goto free_bh;

        /* If ocfs2_read_block() got us a new bh, pass it up. */
        if (!*bh)
                *bh = tmp;

        return rc;

free_bh:
        brelse(tmp);
out:
        return rc;
}

int ocfs2_read_group_descriptor(struct inode *inode, struct ocfs2_dinode *di,
                                u64 gd_blkno, struct buffer_head **bh)
{
        int rc;
        struct buffer_head *tmp = *bh;

        rc = ocfs2_read_block(INODE_CACHE(inode), gd_blkno, &tmp,
                              ocfs2_validate_group_descriptor);
        if (rc)
                goto out;

        rc = ocfs2_validate_gd_parent(inode->i_sb, di, tmp, 0);
        if (rc) {
                brelse(tmp);
                goto out;
        }

        /* If ocfs2_read_block() got us a new bh, pass it up. */
        if (!*bh)
                *bh = tmp;

out:
        return rc;
}

static void ocfs2_bg_discontig_add_extent(struct ocfs2_super *osb,
                                          struct ocfs2_group_desc *bg,
                                          struct ocfs2_chain_list *cl,
                                          u64 p_blkno, unsigned int clusters)
{
        struct ocfs2_extent_list *el = &bg->bg_list;
        struct ocfs2_extent_rec *rec;

        BUG_ON(!ocfs2_supports_discontig_bg(osb));
        if (!el->l_next_free_rec)
                el->l_count = cpu_to_le16(ocfs2_extent_recs_per_gd(osb->sb));
        rec = &el->l_recs[le16_to_cpu(el->l_next_free_rec)];
        rec->e_blkno = cpu_to_le64(p_blkno);
        rec->e_cpos = cpu_to_le32(le16_to_cpu(bg->bg_bits) /
                                  le16_to_cpu(cl->cl_bpc));
        rec->e_leaf_clusters = cpu_to_le16(clusters);
        le16_add_cpu(&bg->bg_bits, clusters * le16_to_cpu(cl->cl_bpc));
        le16_add_cpu(&bg->bg_free_bits_count,
                     clusters * le16_to_cpu(cl->cl_bpc));
        le16_add_cpu(&el->l_next_free_rec, 1);
}

static int ocfs2_block_group_fill(handle_t *handle,
                                  struct inode *alloc_inode,
                                  struct buffer_head *bg_bh,
                                  u64 group_blkno,
                                  unsigned int group_clusters,
                                  u16 my_chain,
                                  struct ocfs2_chain_list *cl)
{
        int status = 0;
        struct ocfs2_super *osb = OCFS2_SB(alloc_inode->i_sb);
        struct ocfs2_group_desc *bg = (struct ocfs2_group_desc *) bg_bh->b_data;
        struct super_block * sb = alloc_inode->i_sb;

        if (((unsigned long long) bg_bh->b_blocknr) != group_blkno) {
                status = ocfs2_error(alloc_inode->i_sb,
                                     "group block (%llu) != b_blocknr (%llu)\n",
                                     (unsigned long long)group_blkno,
                                     (unsigned long long) bg_bh->b_blocknr);
                goto bail;
        }

        status = ocfs2_journal_access_gd(handle,
                                         INODE_CACHE(alloc_inode),
                                         bg_bh,
                                         OCFS2_JOURNAL_ACCESS_CREATE);
        if (status < 0) {
                mlog_errno(status);
                goto bail;
        }

        memset(bg, 0, sb->s_blocksize);
        strscpy(bg->bg_signature, OCFS2_GROUP_DESC_SIGNATURE);
        bg->bg_generation = cpu_to_le32(osb->fs_generation);
        bg->bg_size = cpu_to_le16(ocfs2_group_bitmap_size(sb, 1,
                                                osb->s_feature_incompat));
        bg->bg_chain = cpu_to_le16(my_chain);
        bg->bg_next_group = cl->cl_recs[my_chain].c_blkno;
        bg->bg_parent_dinode = cpu_to_le64(OCFS2_I(alloc_inode)->ip_blkno);
        bg->bg_blkno = cpu_to_le64(group_blkno);
        if (group_clusters == le16_to_cpu(cl->cl_cpg))
                bg->bg_bits = cpu_to_le16(ocfs2_bits_per_group(cl));
        else
                ocfs2_bg_discontig_add_extent(osb, bg, cl, group_blkno,
                                              group_clusters);

        /* set the 1st bit in the bitmap to account for the descriptor block */
        ocfs2_set_bit(0, (unsigned long *)bg->bg_bitmap);
        bg->bg_free_bits_count = cpu_to_le16(le16_to_cpu(bg->bg_bits) - 1);

        ocfs2_journal_dirty(handle, bg_bh);

        /* There is no need to zero out or otherwise initialize the
         * other blocks in a group - All valid FS metadata in a block
         * group stores the superblock fs_generation value at
         * allocation time. */

bail:
        if (status)
                mlog_errno(status);
        return status;
}

static inline u16 ocfs2_find_smallest_chain(struct ocfs2_chain_list *cl)
{
        u16 curr, best;

        best = curr = 0;
        while (curr < le16_to_cpu(cl->cl_count)) {
                if (le32_to_cpu(cl->cl_recs[best].c_total) >
                    le32_to_cpu(cl->cl_recs[curr].c_total))
                        best = curr;
                curr++;
        }
        return best;
}

static struct buffer_head *
ocfs2_block_group_alloc_contig(struct ocfs2_super *osb, handle_t *handle,
                               struct inode *alloc_inode,
                               struct ocfs2_alloc_context *ac,
                               struct ocfs2_chain_list *cl)
{
        int status;
        u32 bit_off, num_bits;
        u64 bg_blkno;
        struct buffer_head *bg_bh;
        unsigned int alloc_rec = ocfs2_find_smallest_chain(cl);

        status = ocfs2_claim_clusters(handle, ac,
                                      le16_to_cpu(cl->cl_cpg), &bit_off,
                                      &num_bits);
        if (status < 0) {
                if (status != -ENOSPC)
                        mlog_errno(status);
                goto bail;
        }

        /* setup the group */
        bg_blkno = ocfs2_clusters_to_blocks(osb->sb, bit_off);
        trace_ocfs2_block_group_alloc_contig(
             (unsigned long long)bg_blkno, alloc_rec);

        bg_bh = sb_getblk(osb->sb, bg_blkno);
        if (!bg_bh) {
                status = -ENOMEM;
                mlog_errno(status);
                goto bail;
        }
        ocfs2_set_new_buffer_uptodate(INODE_CACHE(alloc_inode), bg_bh);

        status = ocfs2_block_group_fill(handle, alloc_inode, bg_bh,
                                        bg_blkno, num_bits, alloc_rec, cl);
        if (status < 0) {
                brelse(bg_bh);
                mlog_errno(status);
        }

bail:
        return status ? ERR_PTR(status) : bg_bh;
}

static int ocfs2_block_group_claim_bits(struct ocfs2_super *osb,
                                        handle_t *handle,
                                        struct ocfs2_alloc_context *ac,
                                        unsigned int min_bits,
                                        u32 *bit_off, u32 *num_bits)
{
        int status = 0;

        while (min_bits) {
                status = ocfs2_claim_clusters(handle, ac, min_bits,
                                              bit_off, num_bits);
                if (status != -ENOSPC)
                        break;

                min_bits >>= 1;
        }

        return status;
}

static int ocfs2_block_group_grow_discontig(handle_t *handle,
                                            struct inode *alloc_inode,
                                            struct buffer_head *bg_bh,
                                            struct ocfs2_alloc_context *ac,
                                            struct ocfs2_chain_list *cl,
                                            unsigned int min_bits)
{
        int status;
        struct ocfs2_super *osb = OCFS2_SB(alloc_inode->i_sb);
        struct ocfs2_group_desc *bg =
                (struct ocfs2_group_desc *)bg_bh->b_data;
        unsigned int needed = le16_to_cpu(cl->cl_cpg) -
                         le16_to_cpu(bg->bg_bits) / le16_to_cpu(cl->cl_bpc);
        u32 p_cpos, clusters;
        u64 p_blkno;
        struct ocfs2_extent_list *el = &bg->bg_list;

        status = ocfs2_journal_access_gd(handle,
                                         INODE_CACHE(alloc_inode),
                                         bg_bh,
                                         OCFS2_JOURNAL_ACCESS_CREATE);
        if (status < 0) {
                mlog_errno(status);
                goto bail;
        }

        while ((needed > 0) && (le16_to_cpu(el->l_next_free_rec) <
                                le16_to_cpu(el->l_count))) {
                if (min_bits > needed)
                        min_bits = needed;
                status = ocfs2_block_group_claim_bits(osb, handle, ac,
                                                      min_bits, &p_cpos,
                                                      &clusters);
                if (status < 0) {
                        if (status != -ENOSPC)
                                mlog_errno(status);
                        goto bail;
                }
                p_blkno = ocfs2_clusters_to_blocks(osb->sb, p_cpos);
                ocfs2_bg_discontig_add_extent(osb, bg, cl, p_blkno,
                                              clusters);

                min_bits = clusters;
                needed = le16_to_cpu(cl->cl_cpg) -
                         le16_to_cpu(bg->bg_bits) / le16_to_cpu(cl->cl_bpc);
        }

        if (needed > 0) {
                /*
                 * We have used up all the extent rec but can't fill up
                 * the cpg. So bail out.
                 */
                status = -ENOSPC;
                goto bail;
        }

        ocfs2_journal_dirty(handle, bg_bh);

bail:
        return status;
}

static void ocfs2_bg_alloc_cleanup(handle_t *handle,
                                   struct ocfs2_alloc_context *cluster_ac,
                                   struct inode *alloc_inode,
                                   struct buffer_head *bg_bh)
{
        int i, ret;
        struct ocfs2_group_desc *bg;
        struct ocfs2_extent_list *el;
        struct ocfs2_extent_rec *rec;

        if (!bg_bh)
                return;

        bg = (struct ocfs2_group_desc *)bg_bh->b_data;
        el = &bg->bg_list;
        for (i = 0; i < le16_to_cpu(el->l_next_free_rec); i++) {
                rec = &el->l_recs[i];
                ret = ocfs2_free_clusters(handle, cluster_ac->ac_inode,
                                          cluster_ac->ac_bh,
                                          le64_to_cpu(rec->e_blkno),
                                          le16_to_cpu(rec->e_leaf_clusters));
                if (ret)
                        mlog_errno(ret);
                /* Try all the clusters to free */
        }

        ocfs2_remove_from_cache(INODE_CACHE(alloc_inode), bg_bh);
        brelse(bg_bh);
}

static struct buffer_head *
ocfs2_block_group_alloc_discontig(handle_t *handle,
                                  struct inode *alloc_inode,
                                  struct ocfs2_alloc_context *ac,
                                  struct ocfs2_chain_list *cl)
{
        int status;
        u32 bit_off, num_bits;
        u64 bg_blkno;
        unsigned int min_bits = le16_to_cpu(cl->cl_cpg) >> 1;
        struct buffer_head *bg_bh = NULL;
        unsigned int alloc_rec = ocfs2_find_smallest_chain(cl);
        struct ocfs2_super *osb = OCFS2_SB(alloc_inode->i_sb);

        if (!ocfs2_supports_discontig_bg(osb)) {
                status = -ENOSPC;
                goto bail;
        }

        status = ocfs2_extend_trans(handle,
                                    ocfs2_calc_bg_discontig_credits(osb->sb));
        if (status) {
                mlog_errno(status);
                goto bail;
        }

        /*
         * We're going to be grabbing from multiple cluster groups.
         * We don't have enough credits to relink them all, and the
         * cluster groups will be staying in cache for the duration of
         * this operation.
         */
        ac->ac_disable_chain_relink = 1;

        /* Claim the first region */
        status = ocfs2_block_group_claim_bits(osb, handle, ac, min_bits,
                                              &bit_off, &num_bits);
        if (status < 0) {
                if (status != -ENOSPC)
                        mlog_errno(status);
                goto bail;
        }
        min_bits = num_bits;

        /* setup the group */
        bg_blkno = ocfs2_clusters_to_blocks(osb->sb, bit_off);
        trace_ocfs2_block_group_alloc_discontig(
                                (unsigned long long)bg_blkno, alloc_rec);

        bg_bh = sb_getblk(osb->sb, bg_blkno);
        if (!bg_bh) {
                status = -ENOMEM;
                mlog_errno(status);
                goto bail;
        }
        ocfs2_set_new_buffer_uptodate(INODE_CACHE(alloc_inode), bg_bh);

        status = ocfs2_block_group_fill(handle, alloc_inode, bg_bh,
                                        bg_blkno, num_bits, alloc_rec, cl);
        if (status < 0) {
                mlog_errno(status);
                goto bail;
        }

        status = ocfs2_block_group_grow_discontig(handle, alloc_inode,
                                                  bg_bh, ac, cl, min_bits);
        if (status)
                mlog_errno(status);

bail:
        if (status)
                ocfs2_bg_alloc_cleanup(handle, ac, alloc_inode, bg_bh);
        return status ? ERR_PTR(status) : bg_bh;
}

/*
 * We expect the block group allocator to already be locked.
 */
static int ocfs2_block_group_alloc(struct ocfs2_super *osb,
                                   struct inode *alloc_inode,
                                   struct buffer_head *bh,
                                   u64 max_block,
                                   u64 *last_alloc_group,
                                   int flags)
{
        int status, credits;
        struct ocfs2_dinode *fe = (struct ocfs2_dinode *) bh->b_data;
        struct ocfs2_chain_list *cl;
        struct ocfs2_alloc_context *ac = NULL;
        handle_t *handle = NULL;
        u16 alloc_rec;
        struct buffer_head *bg_bh = NULL;
        struct ocfs2_group_desc *bg;

        BUG_ON(ocfs2_is_cluster_bitmap(alloc_inode));

        cl = &fe->id2.i_chain;
        status = ocfs2_reserve_clusters_with_limit(osb,
                                                   le16_to_cpu(cl->cl_cpg),
                                                   max_block, flags, &ac);
        if (status < 0) {
                if (status != -ENOSPC)
                        mlog_errno(status);
                goto bail;
        }

        credits = ocfs2_calc_group_alloc_credits(osb->sb,
                                                 le16_to_cpu(cl->cl_cpg));
        handle = ocfs2_start_trans(osb, credits);
        if (IS_ERR(handle)) {
                status = PTR_ERR(handle);
                handle = NULL;
                mlog_errno(status);
                goto bail;
        }

        if (last_alloc_group && *last_alloc_group != 0) {
                trace_ocfs2_block_group_alloc(
                                (unsigned long long)*last_alloc_group);
                ac->ac_last_group = *last_alloc_group;
        }

        bg_bh = ocfs2_block_group_alloc_contig(osb, handle, alloc_inode,
                                               ac, cl);
        if (PTR_ERR(bg_bh) == -ENOSPC) {
                ac->ac_which = OCFS2_AC_USE_MAIN_DISCONTIG;
                bg_bh = ocfs2_block_group_alloc_discontig(handle,
                                                          alloc_inode,
                                                          ac, cl);
        }
        if (IS_ERR(bg_bh)) {
                status = PTR_ERR(bg_bh);
                bg_bh = NULL;
                if (status != -ENOSPC)
                        mlog_errno(status);
                goto bail;
        }
        bg = (struct ocfs2_group_desc *) bg_bh->b_data;

        status = ocfs2_journal_access_di(handle, INODE_CACHE(alloc_inode),
                                         bh, OCFS2_JOURNAL_ACCESS_WRITE);
        if (status < 0) {
                mlog_errno(status);
                goto bail;
        }

        alloc_rec = le16_to_cpu(bg->bg_chain);
        le32_add_cpu(&cl->cl_recs[alloc_rec].c_free,
                     le16_to_cpu(bg->bg_free_bits_count));
        le32_add_cpu(&cl->cl_recs[alloc_rec].c_total,
                     le16_to_cpu(bg->bg_bits));
        cl->cl_recs[alloc_rec].c_blkno = bg->bg_blkno;
        if (le16_to_cpu(cl->cl_next_free_rec) < le16_to_cpu(cl->cl_count))
                le16_add_cpu(&cl->cl_next_free_rec, 1);

        le32_add_cpu(&fe->id1.bitmap1.i_used, le16_to_cpu(bg->bg_bits) -
                                        le16_to_cpu(bg->bg_free_bits_count));
        le32_add_cpu(&fe->id1.bitmap1.i_total, le16_to_cpu(bg->bg_bits));
        le32_add_cpu(&fe->i_clusters, le16_to_cpu(cl->cl_cpg));

        ocfs2_journal_dirty(handle, bh);

        spin_lock(&OCFS2_I(alloc_inode)->ip_lock);
        OCFS2_I(alloc_inode)->ip_clusters = le32_to_cpu(fe->i_clusters);
        fe->i_size = cpu_to_le64(ocfs2_clusters_to_bytes(alloc_inode->i_sb,
                                             le32_to_cpu(fe->i_clusters)));
        spin_unlock(&OCFS2_I(alloc_inode)->ip_lock);
        i_size_write(alloc_inode, le64_to_cpu(fe->i_size));
        alloc_inode->i_blocks = ocfs2_inode_sector_count(alloc_inode);
        ocfs2_update_inode_fsync_trans(handle, alloc_inode, 0);

        status = 0;

        /* save the new last alloc group so that the caller can cache it. */
        if (last_alloc_group)
                *last_alloc_group = ac->ac_last_group;

bail:
        if (handle)
                ocfs2_commit_trans(osb, handle);

        if (ac)
                ocfs2_free_alloc_context(ac);

        brelse(bg_bh);

        if (status)
                mlog_errno(status);
        return status;
}

static int ocfs2_reserve_suballoc_bits(struct ocfs2_super *osb,
                                       struct ocfs2_alloc_context *ac,
                                       int type,
                                       u32 slot,
                                       u64 *last_alloc_group,
                                       int flags)
{
        int status;
        u32 bits_wanted = ac->ac_bits_wanted;
        struct inode *alloc_inode;
        struct buffer_head *bh = NULL;
        struct ocfs2_dinode *fe;
        u32 free_bits;

        alloc_inode = ocfs2_get_system_file_inode(osb, type, slot);
        if (!alloc_inode) {
                mlog_errno(-EINVAL);
                return -EINVAL;
        }

        inode_lock(alloc_inode);

        status = ocfs2_inode_lock(alloc_inode, &bh, 1);
        if (status < 0) {
                inode_unlock(alloc_inode);
                iput(alloc_inode);

                mlog_errno(status);
                return status;
        }

        ac->ac_inode = alloc_inode;
        ac->ac_alloc_slot = slot;

        fe = (struct ocfs2_dinode *) bh->b_data;

        /* The bh was validated by the inode read inside
         * ocfs2_inode_lock().  Any corruption is a code bug. */
        BUG_ON(!OCFS2_IS_VALID_DINODE(fe));

        if (!(fe->i_flags & cpu_to_le32(OCFS2_CHAIN_FL))) {
                status = ocfs2_error(alloc_inode->i_sb,
                                     "Invalid chain allocator %llu\n",
                                     (unsigned long long)le64_to_cpu(fe->i_blkno));
                goto bail;
        }

        free_bits = le32_to_cpu(fe->id1.bitmap1.i_total) -
                le32_to_cpu(fe->id1.bitmap1.i_used);

        if (bits_wanted > free_bits) {
                /* cluster bitmap never grows */
                if (ocfs2_is_cluster_bitmap(alloc_inode)) {
                        trace_ocfs2_reserve_suballoc_bits_nospc(bits_wanted,
                                                                free_bits);
                        status = -ENOSPC;
                        goto bail;
                }

                if (!(flags & ALLOC_NEW_GROUP)) {
                        trace_ocfs2_reserve_suballoc_bits_no_new_group(
                                                slot, bits_wanted, free_bits);
                        status = -ENOSPC;
                        goto bail;
                }

                status = ocfs2_block_group_alloc(osb, alloc_inode, bh,
                                                 ac->ac_max_block,
                                                 last_alloc_group, flags);
                if (status < 0) {
                        if (status != -ENOSPC)
                                mlog_errno(status);
                        goto bail;
                }
                atomic_inc(&osb->alloc_stats.bg_extends);

                /* You should never ask for this much metadata */
                BUG_ON(bits_wanted >
                       (le32_to_cpu(fe->id1.bitmap1.i_total)
                        - le32_to_cpu(fe->id1.bitmap1.i_used)));
        }

        get_bh(bh);
        ac->ac_bh = bh;
bail:
        brelse(bh);

        if (status)
                mlog_errno(status);
        return status;
}

static void ocfs2_init_inode_steal_slot(struct ocfs2_super *osb)
{
        spin_lock(&osb->osb_lock);
        osb->s_inode_steal_slot = OCFS2_INVALID_SLOT;
        spin_unlock(&osb->osb_lock);
        atomic_set(&osb->s_num_inodes_stolen, 0);
}

static void ocfs2_init_meta_steal_slot(struct ocfs2_super *osb)
{
        spin_lock(&osb->osb_lock);
        osb->s_meta_steal_slot = OCFS2_INVALID_SLOT;
        spin_unlock(&osb->osb_lock);
        atomic_set(&osb->s_num_meta_stolen, 0);
}

void ocfs2_init_steal_slots(struct ocfs2_super *osb)
{
        ocfs2_init_inode_steal_slot(osb);
        ocfs2_init_meta_steal_slot(osb);
}

static void __ocfs2_set_steal_slot(struct ocfs2_super *osb, int slot, int type)
{
        spin_lock(&osb->osb_lock);
        if (type == INODE_ALLOC_SYSTEM_INODE)
                osb->s_inode_steal_slot = (u16)slot;
        else if (type == EXTENT_ALLOC_SYSTEM_INODE)
                osb->s_meta_steal_slot = (u16)slot;
        spin_unlock(&osb->osb_lock);
}

static int __ocfs2_get_steal_slot(struct ocfs2_super *osb, int type)
{
        int slot = OCFS2_INVALID_SLOT;

        spin_lock(&osb->osb_lock);
        if (type == INODE_ALLOC_SYSTEM_INODE)
                slot = osb->s_inode_steal_slot;
        else if (type == EXTENT_ALLOC_SYSTEM_INODE)
                slot = osb->s_meta_steal_slot;
        spin_unlock(&osb->osb_lock);

        return slot;
}

static int ocfs2_get_inode_steal_slot(struct ocfs2_super *osb)
{
        return __ocfs2_get_steal_slot(osb, INODE_ALLOC_SYSTEM_INODE);
}

static int ocfs2_get_meta_steal_slot(struct ocfs2_super *osb)
{
        return __ocfs2_get_steal_slot(osb, EXTENT_ALLOC_SYSTEM_INODE);
}

static int ocfs2_steal_resource(struct ocfs2_super *osb,
                                struct ocfs2_alloc_context *ac,
                                int type)
{
        int i, status = -ENOSPC;
        int slot = __ocfs2_get_steal_slot(osb, type);

        /* Start to steal resource from the first slot after ours. */
        if (slot == OCFS2_INVALID_SLOT)
                slot = osb->slot_num + 1;

        for (i = 0; i < osb->max_slots; i++, slot++) {
                if (slot == osb->max_slots)
                        slot = 0;

                if (slot == osb->slot_num)
                        continue;

                status = ocfs2_reserve_suballoc_bits(osb, ac,
                                                     type,
                                                     (u32)slot, NULL,
                                                     NOT_ALLOC_NEW_GROUP);
                if (status >= 0) {
                        __ocfs2_set_steal_slot(osb, slot, type);
                        break;
                }

                ocfs2_free_ac_resource(ac);
        }

        return status;
}

static int ocfs2_steal_inode(struct ocfs2_super *osb,
                             struct ocfs2_alloc_context *ac)
{
        return ocfs2_steal_resource(osb, ac, INODE_ALLOC_SYSTEM_INODE);
}

static int ocfs2_steal_meta(struct ocfs2_super *osb,
                            struct ocfs2_alloc_context *ac)
{
        return ocfs2_steal_resource(osb, ac, EXTENT_ALLOC_SYSTEM_INODE);
}

int ocfs2_reserve_new_metadata_blocks(struct ocfs2_super *osb,
                                      int blocks,
                                      struct ocfs2_alloc_context **ac)
{
        int status;
        int slot = ocfs2_get_meta_steal_slot(osb);

        *ac = kzalloc_obj(struct ocfs2_alloc_context);
        if (!(*ac)) {
                status = -ENOMEM;
                mlog_errno(status);
                goto bail;
        }

        (*ac)->ac_bits_wanted = blocks;
        (*ac)->ac_which = OCFS2_AC_USE_META;
        (*ac)->ac_group_search = ocfs2_block_group_search;

        if (slot != OCFS2_INVALID_SLOT &&
                atomic_read(&osb->s_num_meta_stolen) < OCFS2_MAX_TO_STEAL)
                goto extent_steal;

        atomic_set(&osb->s_num_meta_stolen, 0);
        status = ocfs2_reserve_suballoc_bits(osb, (*ac),
                                             EXTENT_ALLOC_SYSTEM_INODE,
                                             (u32)osb->slot_num, NULL,
                                             ALLOC_GROUPS_FROM_GLOBAL|ALLOC_NEW_GROUP);


        if (status >= 0) {
                status = 0;
                if (slot != OCFS2_INVALID_SLOT)
                        ocfs2_init_meta_steal_slot(osb);
                goto bail;
        } else if (status < 0 && status != -ENOSPC) {
                mlog_errno(status);
                goto bail;
        }

        ocfs2_free_ac_resource(*ac);

extent_steal:
        status = ocfs2_steal_meta(osb, *ac);
        atomic_inc(&osb->s_num_meta_stolen);
        if (status < 0) {
                if (status != -ENOSPC)
                        mlog_errno(status);
                goto bail;
        }

        status = 0;
bail:
        if ((status < 0) && *ac) {
                ocfs2_free_alloc_context(*ac);
                *ac = NULL;
        }

        if (status)
                mlog_errno(status);
        return status;
}

int ocfs2_reserve_new_metadata(struct ocfs2_super *osb,
                               struct ocfs2_extent_list *root_el,
                               struct ocfs2_alloc_context **ac)
{
        return ocfs2_reserve_new_metadata_blocks(osb,
                                        ocfs2_extend_meta_needed(root_el),
                                        ac);
}

int ocfs2_reserve_new_inode(struct ocfs2_super *osb,
                            struct ocfs2_alloc_context **ac)
{
        int status;
        int slot = ocfs2_get_inode_steal_slot(osb);
        u64 alloc_group;

        *ac = kzalloc_obj(struct ocfs2_alloc_context);
        if (!(*ac)) {
                status = -ENOMEM;
                mlog_errno(status);
                goto bail;
        }

        (*ac)->ac_bits_wanted = 1;
        (*ac)->ac_which = OCFS2_AC_USE_INODE;

        (*ac)->ac_group_search = ocfs2_block_group_search;

        /*
         * stat(2) can't handle i_ino > 32bits, so we tell the
         * lower levels not to allocate us a block group past that
         * limit.  The 'inode64' mount option avoids this behavior.
         */
        if (!(osb->s_mount_opt & OCFS2_MOUNT_INODE64))
                (*ac)->ac_max_block = (u32)~0U;

        /*
         * slot is set when we successfully steal inode from other nodes.
         * It is reset in 3 places:
         * 1. when we flush the truncate log
         * 2. when we complete local alloc recovery.
         * 3. when we successfully allocate from our own slot.
         * After it is set, we will go on stealing inodes until we find the
         * need to check our slots to see whether there is some space for us.
         */
        if (slot != OCFS2_INVALID_SLOT &&
            atomic_read(&osb->s_num_inodes_stolen) < OCFS2_MAX_TO_STEAL)
                goto inode_steal;

        atomic_set(&osb->s_num_inodes_stolen, 0);
        alloc_group = osb->osb_inode_alloc_group;
        status = ocfs2_reserve_suballoc_bits(osb, *ac,
                                             INODE_ALLOC_SYSTEM_INODE,
                                             (u32)osb->slot_num,
                                             &alloc_group,
                                             ALLOC_NEW_GROUP |
                                             ALLOC_GROUPS_FROM_GLOBAL);
        if (status >= 0) {
                status = 0;

                spin_lock(&osb->osb_lock);
                osb->osb_inode_alloc_group = alloc_group;
                spin_unlock(&osb->osb_lock);
                trace_ocfs2_reserve_new_inode_new_group(
                        (unsigned long long)alloc_group);

                /*
                 * Some inodes must be freed by us, so try to allocate
                 * from our own next time.
                 */
                if (slot != OCFS2_INVALID_SLOT)
                        ocfs2_init_inode_steal_slot(osb);
                goto bail;
        } else if (status < 0 && status != -ENOSPC) {
                mlog_errno(status);
                goto bail;
        }

        ocfs2_free_ac_resource(*ac);

inode_steal:
        status = ocfs2_steal_inode(osb, *ac);
        atomic_inc(&osb->s_num_inodes_stolen);
        if (status < 0) {
                if (status != -ENOSPC)
                        mlog_errno(status);
                goto bail;
        }

        status = 0;
bail:
        if ((status < 0) && *ac) {
                ocfs2_free_alloc_context(*ac);
                *ac = NULL;
        }

        if (status)
                mlog_errno(status);
        return status;
}

/* local alloc code has to do the same thing, so rather than do this
 * twice.. */
int ocfs2_reserve_cluster_bitmap_bits(struct ocfs2_super *osb,
                                      struct ocfs2_alloc_context *ac)
{
        int status;

        ac->ac_which = OCFS2_AC_USE_MAIN;
        ac->ac_group_search = ocfs2_cluster_group_search;

        status = ocfs2_reserve_suballoc_bits(osb, ac,
                                             GLOBAL_BITMAP_SYSTEM_INODE,
                                             OCFS2_INVALID_SLOT, NULL,
                                             ALLOC_NEW_GROUP);
        if (status < 0 && status != -ENOSPC)
                mlog_errno(status);

        return status;
}

/* Callers don't need to care which bitmap (local alloc or main) to
 * use so we figure it out for them, but unfortunately this clutters
 * things a bit. */
static int ocfs2_reserve_clusters_with_limit(struct ocfs2_super *osb,
                                             u32 bits_wanted, u64 max_block,
                                             int flags,
                                             struct ocfs2_alloc_context **ac)
{
        int status, ret = 0;
        int retried = 0;

        *ac = kzalloc_obj(struct ocfs2_alloc_context);
        if (!(*ac)) {
                status = -ENOMEM;
                mlog_errno(status);
                goto bail;
        }

        (*ac)->ac_bits_wanted = bits_wanted;
        (*ac)->ac_max_block = max_block;

        status = -ENOSPC;
        if (!(flags & ALLOC_GROUPS_FROM_GLOBAL) &&
            ocfs2_alloc_should_use_local(osb, bits_wanted)) {
                status = ocfs2_reserve_local_alloc_bits(osb,
                                                        bits_wanted,
                                                        *ac);
                if ((status < 0) && (status != -ENOSPC)) {
                        mlog_errno(status);
                        goto bail;
                }
        }

        if (status == -ENOSPC) {
retry:
                status = ocfs2_reserve_cluster_bitmap_bits(osb, *ac);
                /* Retry if there is sufficient space cached in truncate log */
                if (status == -ENOSPC && !retried) {
                        retried = 1;
                        ocfs2_inode_unlock((*ac)->ac_inode, 1);
                        inode_unlock((*ac)->ac_inode);

                        ret = ocfs2_try_to_free_truncate_log(osb, bits_wanted);
                        if (ret == 1) {
                                iput((*ac)->ac_inode);
                                (*ac)->ac_inode = NULL;
                                goto retry;
                        }

                        if (ret < 0)
                                mlog_errno(ret);

                        inode_lock((*ac)->ac_inode);
                        ret = ocfs2_inode_lock((*ac)->ac_inode, NULL, 1);
                        if (ret < 0) {
                                mlog_errno(ret);
                                inode_unlock((*ac)->ac_inode);
                                iput((*ac)->ac_inode);
                                (*ac)->ac_inode = NULL;
                                goto bail;
                        }
                }
                if (status < 0) {
                        if (status != -ENOSPC)
                                mlog_errno(status);
                        goto bail;
                }
        }

        status = 0;
bail:
        if ((status < 0) && *ac) {
                ocfs2_free_alloc_context(*ac);
                *ac = NULL;
        }

        if (status)
                mlog_errno(status);
        return status;
}

int ocfs2_reserve_clusters(struct ocfs2_super *osb,
                           u32 bits_wanted,
                           struct ocfs2_alloc_context **ac)
{
        return ocfs2_reserve_clusters_with_limit(osb, bits_wanted, 0,
                                                 ALLOC_NEW_GROUP, ac);
}

/*
 * More or less lifted from ext3. I'll leave their description below:
 *
 * "For ext3 allocations, we must not reuse any blocks which are
 * allocated in the bitmap buffer's "last committed data" copy.  This
 * prevents deletes from freeing up the page for reuse until we have
 * committed the delete transaction.
 *
 * If we didn't do this, then deleting something and reallocating it as
 * data would allow the old block to be overwritten before the
 * transaction committed (because we force data to disk before commit).
 * This would lead to corruption if we crashed between overwriting the
 * data and committing the delete.
 *
 * @@@ We may want to make this allocation behaviour conditional on
 * data-writes at some point, and disable it for metadata allocations or
 * sync-data inodes."
 *
 * Note: OCFS2 already does this differently for metadata vs data
 * allocations, as those bitmaps are separate and undo access is never
 * called on a metadata group descriptor.
 */
static int ocfs2_test_bg_bit_allocatable(struct buffer_head *bg_bh,
                                         int nr)
{
        struct ocfs2_group_desc *bg = (struct ocfs2_group_desc *) bg_bh->b_data;
        struct journal_head *jh;
        int ret;

        if (ocfs2_test_bit(nr, (unsigned long *)bg->bg_bitmap))
                return 0;

        jh = jbd2_journal_grab_journal_head(bg_bh);
        if (!jh)
                return 1;

        spin_lock(&jh->b_state_lock);
        bg = (struct ocfs2_group_desc *) jh->b_committed_data;
        if (bg)
                ret = !ocfs2_test_bit(nr, (unsigned long *)bg->bg_bitmap);
        else
                ret = 1;
        spin_unlock(&jh->b_state_lock);
        jbd2_journal_put_journal_head(jh);

        return ret;
}

u16 ocfs2_find_max_contig_free_bits(void *bitmap,
                         u16 total_bits, u16 start)
{
        u16 offset, free_bits;
        u16 contig_bits = 0;

        while (start < total_bits) {
                offset = ocfs2_find_next_zero_bit(bitmap, total_bits, start);
                if (offset == total_bits)
                        break;

                start = ocfs2_find_next_bit(bitmap, total_bits, offset);
                free_bits = start - offset;
                if (contig_bits < free_bits)
                        contig_bits = free_bits;
        }

        return contig_bits;
}

static int ocfs2_block_group_find_clear_bits(struct ocfs2_super *osb,
                                             struct buffer_head *bg_bh,
                                             unsigned int bits_wanted,
                                             unsigned int total_bits,
                                             struct ocfs2_suballoc_result *res)
{
        void *bitmap;
        u16 best_offset, best_size;
        u16 prev_best_size = 0;
        int offset, start, found, status = 0;
        struct ocfs2_group_desc *bg = (struct ocfs2_group_desc *) bg_bh->b_data;

        /* Callers got this descriptor from
         * ocfs2_read_group_descriptor().  Any corruption is a code bug. */
        BUG_ON(!OCFS2_IS_VALID_GROUP_DESC(bg));

        found = start = best_offset = best_size = 0;
        bitmap = bg->bg_bitmap;

        while ((offset = ocfs2_find_next_zero_bit(bitmap, total_bits, start)) <
               total_bits) {
                if (!ocfs2_test_bg_bit_allocatable(bg_bh, offset)) {
                        /* We found a zero, but we can't use it as it
                         * hasn't been put to disk yet! */
                        found = 0;
                        start = offset + 1;
                } else if (offset == start) {
                        /* we found a zero */
                        found++;
                        /* move start to the next bit to test */
                        start++;
                } else {
                        /* got a zero after some ones */
                        found = 1;
                        start = offset + 1;
                        prev_best_size = best_size;
                }
                if (found > best_size) {
                        best_size = found;
                        best_offset = start - found;
                }
                /* we got everything we needed */
                if (found == bits_wanted) {
                        /* mlog(0, "Found it all!\n"); */
                        break;
                }
        }

        /* best_size will be allocated, we save prev_best_size */
        res->sr_max_contig_bits = prev_best_size;
        if (best_size) {
                res->sr_bit_offset = best_offset;
                res->sr_bits = best_size;
        } else {
                status = -ENOSPC;
                /* No error log here -- see the comment above
                 * ocfs2_test_bg_bit_allocatable */
        }

        return status;
}

int ocfs2_block_group_set_bits(handle_t *handle,
                                             struct inode *alloc_inode,
                                             struct ocfs2_group_desc *bg,
                                             struct buffer_head *group_bh,
                                             unsigned int bit_off,
                                             unsigned int num_bits,
                                             unsigned int max_contig_bits,
                                             int fastpath)
{
        int status;
        void *bitmap = bg->bg_bitmap;
        int journal_type = OCFS2_JOURNAL_ACCESS_WRITE;
        unsigned int start = bit_off + num_bits;
        u16 contig_bits;
        struct ocfs2_super *osb = OCFS2_SB(alloc_inode->i_sb);

        /* All callers get the descriptor via
         * ocfs2_read_group_descriptor().  Any corruption is a code bug. */
        BUG_ON(!OCFS2_IS_VALID_GROUP_DESC(bg));
        BUG_ON(le16_to_cpu(bg->bg_free_bits_count) < num_bits);

        trace_ocfs2_block_group_set_bits(bit_off, num_bits);

        if (ocfs2_is_cluster_bitmap(alloc_inode))
                journal_type = OCFS2_JOURNAL_ACCESS_UNDO;

        status = ocfs2_journal_access_gd(handle,
                                         INODE_CACHE(alloc_inode),
                                         group_bh,
                                         journal_type);
        if (status < 0) {
                mlog_errno(status);
                goto bail;
        }

        le16_add_cpu(&bg->bg_free_bits_count, -num_bits);
        if (le16_to_cpu(bg->bg_free_bits_count) > le16_to_cpu(bg->bg_bits)) {
                return ocfs2_error(alloc_inode->i_sb, "Group descriptor # %llu has bit count %u but claims %u are freed. num_bits %d\n",
                                   (unsigned long long)le64_to_cpu(bg->bg_blkno),
                                   le16_to_cpu(bg->bg_bits),
                                   le16_to_cpu(bg->bg_free_bits_count),
                                   num_bits);
        }
        while(num_bits--)
                ocfs2_set_bit(bit_off++, bitmap);

        /*
         * this is optimize path, caller set old contig value
         * in max_contig_bits to bypass finding action.
         */
        if (fastpath) {
                bg->bg_contig_free_bits = cpu_to_le16(max_contig_bits);
        } else if (ocfs2_is_cluster_bitmap(alloc_inode)) {
                /*
                 * Usually, the block group bitmap allocates only 1 bit
                 * at a time, while the cluster group allocates n bits
                 * each time. Therefore, we only save the contig bits for
                 * the cluster group.
                 */
                contig_bits = ocfs2_find_max_contig_free_bits(bitmap,
                                    le16_to_cpu(bg->bg_bits), start);
                if (contig_bits > max_contig_bits)
                        max_contig_bits = contig_bits;
                bg->bg_contig_free_bits = cpu_to_le16(max_contig_bits);
                ocfs2_local_alloc_seen_free_bits(osb, max_contig_bits);
        } else {
                bg->bg_contig_free_bits = 0;
        }

        ocfs2_journal_dirty(handle, group_bh);

bail:
        return status;
}

/* find the one with the most empty bits */
static inline u16 ocfs2_find_victim_chain(struct ocfs2_chain_list *cl)
{
        u16 curr, best;

        BUG_ON(!cl->cl_next_free_rec);

        best = curr = 0;
        while (curr < le16_to_cpu(cl->cl_next_free_rec)) {
                if (le32_to_cpu(cl->cl_recs[curr].c_free) >
                    le32_to_cpu(cl->cl_recs[best].c_free))
                        best = curr;
                curr++;
        }

        BUG_ON(best >= le16_to_cpu(cl->cl_next_free_rec));
        return best;
}

static int ocfs2_relink_block_group(handle_t *handle,
                                    struct inode *alloc_inode,
                                    struct buffer_head *fe_bh,
                                    struct buffer_head *bg_bh,
                                    struct buffer_head *prev_bg_bh,
                                    u16 chain)
{
        int status;
        /* there is a really tiny chance the journal calls could fail,
         * but we wouldn't want inconsistent blocks in *any* case. */
        u64 bg_ptr, prev_bg_ptr;
        struct ocfs2_dinode *fe = (struct ocfs2_dinode *) fe_bh->b_data;
        struct ocfs2_group_desc *bg = (struct ocfs2_group_desc *) bg_bh->b_data;
        struct ocfs2_group_desc *prev_bg = (struct ocfs2_group_desc *) prev_bg_bh->b_data;

        /* The caller got these descriptors from
         * ocfs2_read_group_descriptor().  Any corruption is a code bug. */
        BUG_ON(!OCFS2_IS_VALID_GROUP_DESC(bg));
        BUG_ON(!OCFS2_IS_VALID_GROUP_DESC(prev_bg));

        trace_ocfs2_relink_block_group(
                (unsigned long long)le64_to_cpu(fe->i_blkno), chain,
                (unsigned long long)le64_to_cpu(bg->bg_blkno),
                (unsigned long long)le64_to_cpu(prev_bg->bg_blkno));

        bg_ptr = le64_to_cpu(bg->bg_next_group);
        prev_bg_ptr = le64_to_cpu(prev_bg->bg_next_group);

        status = ocfs2_journal_access_gd(handle, INODE_CACHE(alloc_inode),
                                         prev_bg_bh,
                                         OCFS2_JOURNAL_ACCESS_WRITE);
        if (status < 0)
                goto out;

        prev_bg->bg_next_group = bg->bg_next_group;
        ocfs2_journal_dirty(handle, prev_bg_bh);

        status = ocfs2_journal_access_gd(handle, INODE_CACHE(alloc_inode),
                                         bg_bh, OCFS2_JOURNAL_ACCESS_WRITE);
        if (status < 0)
                goto out_rollback_prev_bg;

        bg->bg_next_group = fe->id2.i_chain.cl_recs[chain].c_blkno;
        ocfs2_journal_dirty(handle, bg_bh);

        status = ocfs2_journal_access_di(handle, INODE_CACHE(alloc_inode),
                                         fe_bh, OCFS2_JOURNAL_ACCESS_WRITE);
        if (status < 0)
                goto out_rollback_bg;

        fe->id2.i_chain.cl_recs[chain].c_blkno = bg->bg_blkno;
        ocfs2_journal_dirty(handle, fe_bh);

out:
        if (status < 0)
                mlog_errno(status);
        return status;

out_rollback_bg:
        bg->bg_next_group = cpu_to_le64(bg_ptr);
out_rollback_prev_bg:
        prev_bg->bg_next_group = cpu_to_le64(prev_bg_ptr);
        goto out;
}

static inline int ocfs2_block_group_reasonably_empty(struct ocfs2_group_desc *bg,
                                                     u32 wanted)
{
        return le16_to_cpu(bg->bg_free_bits_count) > wanted;
}

/* return 0 on success, -ENOSPC to keep searching and any other < 0
 * value on error. */
static int ocfs2_cluster_group_search(struct inode *inode,
                                      struct buffer_head *group_bh,
                                      u32 bits_wanted, u32 min_bits,
                                      u64 max_block,
                                      struct ocfs2_suballoc_result *res)
{
        int search = -ENOSPC;
        int ret;
        u64 blkoff;
        struct ocfs2_group_desc *gd = (struct ocfs2_group_desc *) group_bh->b_data;
        struct ocfs2_super *osb = OCFS2_SB(inode->i_sb);
        unsigned int max_bits, gd_cluster_off;

        BUG_ON(!ocfs2_is_cluster_bitmap(inode));

        if (le16_to_cpu(gd->bg_contig_free_bits) &&
            le16_to_cpu(gd->bg_contig_free_bits) < bits_wanted)
                return -ENOSPC;

        /* ->bg_contig_free_bits may un-initialized, so compare again */
        if (le16_to_cpu(gd->bg_free_bits_count) >= bits_wanted) {
                max_bits = le16_to_cpu(gd->bg_bits);

                /* Tail groups in cluster bitmaps which aren't cpg
                 * aligned are prone to partial extension by a failed
                 * fs resize. If the file system resize never got to
                 * update the dinode cluster count, then we don't want
                 * to trust any clusters past it, regardless of what
                 * the group descriptor says. */
                gd_cluster_off = ocfs2_blocks_to_clusters(inode->i_sb,
                                                          le64_to_cpu(gd->bg_blkno));
                if ((gd_cluster_off + max_bits) >
                    OCFS2_I(inode)->ip_clusters) {
                        max_bits = OCFS2_I(inode)->ip_clusters - gd_cluster_off;
                        trace_ocfs2_cluster_group_search_wrong_max_bits(
                                (unsigned long long)le64_to_cpu(gd->bg_blkno),
                                le16_to_cpu(gd->bg_bits),
                                OCFS2_I(inode)->ip_clusters, max_bits);
                }

                ret = ocfs2_block_group_find_clear_bits(osb,
                                                        group_bh, bits_wanted,
                                                        max_bits, res);
                if (ret)
                        return ret;

                if (max_block) {
                        blkoff = ocfs2_clusters_to_blocks(inode->i_sb,
                                                          gd_cluster_off +
                                                          res->sr_bit_offset +
                                                          res->sr_bits);
                        trace_ocfs2_cluster_group_search_max_block(
                                (unsigned long long)blkoff,
                                (unsigned long long)max_block);
                        if (blkoff > max_block)
                                return -ENOSPC;
                }

                /* ocfs2_block_group_find_clear_bits() might
                 * return success, but we still want to return
                 * -ENOSPC unless it found the minimum number
                 * of bits. */
                if (min_bits <= res->sr_bits)
                        search = 0; /* success */
        }

        return search;
}

static int ocfs2_block_group_search(struct inode *inode,
                                    struct buffer_head *group_bh,
                                    u32 bits_wanted, u32 min_bits,
                                    u64 max_block,
                                    struct ocfs2_suballoc_result *res)
{
        int ret = -ENOSPC;
        u64 blkoff;
        struct ocfs2_group_desc *bg = (struct ocfs2_group_desc *) group_bh->b_data;

        BUG_ON(min_bits != 1);
        BUG_ON(ocfs2_is_cluster_bitmap(inode));

        if (le16_to_cpu(bg->bg_free_bits_count) >= bits_wanted) {
                ret = ocfs2_block_group_find_clear_bits(OCFS2_SB(inode->i_sb),
                                                        group_bh, bits_wanted,
                                                        le16_to_cpu(bg->bg_bits),
                                                        res);
                if (!ret && max_block) {
                        blkoff = le64_to_cpu(bg->bg_blkno) +
                                res->sr_bit_offset + res->sr_bits;
                        trace_ocfs2_block_group_search_max_block(
                                (unsigned long long)blkoff,
                                (unsigned long long)max_block);
                        if (blkoff > max_block)
                                ret = -ENOSPC;
                }
        }

        return ret;
}

int ocfs2_alloc_dinode_update_counts(struct inode *inode,
                                       handle_t *handle,
                                       struct buffer_head *di_bh,
                                       u32 num_bits,
                                       u16 chain)
{
        int ret;
        u32 tmp_used;
        struct ocfs2_dinode *di = (struct ocfs2_dinode *) di_bh->b_data;
        struct ocfs2_chain_list *cl = (struct ocfs2_chain_list *) &di->id2.i_chain;

        ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), di_bh,
                                      OCFS2_JOURNAL_ACCESS_WRITE);
        if (ret < 0) {
                mlog_errno(ret);
                goto out;
        }

        tmp_used = le32_to_cpu(di->id1.bitmap1.i_used);
        di->id1.bitmap1.i_used = cpu_to_le32(num_bits + tmp_used);
        le32_add_cpu(&cl->cl_recs[chain].c_free, -num_bits);
        ocfs2_journal_dirty(handle, di_bh);

out:
        return ret;
}

void ocfs2_rollback_alloc_dinode_counts(struct inode *inode,
                                       struct buffer_head *di_bh,
                                       u32 num_bits,
                                       u16 chain)
{
        u32 tmp_used;
        struct ocfs2_dinode *di = (struct ocfs2_dinode *) di_bh->b_data;
        struct ocfs2_chain_list *cl;

        cl = (struct ocfs2_chain_list *)&di->id2.i_chain;
        tmp_used = le32_to_cpu(di->id1.bitmap1.i_used);
        di->id1.bitmap1.i_used = cpu_to_le32(tmp_used - num_bits);
        le32_add_cpu(&cl->cl_recs[chain].c_free, num_bits);
}

static int ocfs2_bg_discontig_fix_by_rec(struct ocfs2_suballoc_result *res,
                                         struct ocfs2_extent_rec *rec,
                                         struct ocfs2_chain_list *cl)
{
        unsigned int bpc = le16_to_cpu(cl->cl_bpc);
        unsigned int bitoff = le32_to_cpu(rec->e_cpos) * bpc;
        unsigned int bitcount = le16_to_cpu(rec->e_leaf_clusters) * bpc;

        if (res->sr_bit_offset < bitoff)
                return 0;
        if (res->sr_bit_offset >= (bitoff + bitcount))
                return 0;
        res->sr_blkno = le64_to_cpu(rec->e_blkno) +
                (res->sr_bit_offset - bitoff);
        if ((res->sr_bit_offset + res->sr_bits) > (bitoff + bitcount))
                res->sr_bits = (bitoff + bitcount) - res->sr_bit_offset;
        return 1;
}

static void ocfs2_bg_discontig_fix_result(struct ocfs2_alloc_context *ac,
                                          struct ocfs2_group_desc *bg,
                                          struct ocfs2_suballoc_result *res)
{
        int i;
        u64 bg_blkno = res->sr_bg_blkno;  /* Save off */
        struct ocfs2_extent_rec *rec;
        struct ocfs2_dinode *di = (struct ocfs2_dinode *)ac->ac_bh->b_data;
        struct ocfs2_chain_list *cl = &di->id2.i_chain;

        if (ocfs2_is_cluster_bitmap(ac->ac_inode)) {
                res->sr_blkno = 0;
                return;
        }

        res->sr_blkno = res->sr_bg_blkno + res->sr_bit_offset;
        res->sr_bg_blkno = 0;  /* Clear it for contig block groups */
        if (!ocfs2_supports_discontig_bg(OCFS2_SB(ac->ac_inode->i_sb)) ||
            !bg->bg_list.l_next_free_rec)
                return;

        for (i = 0; i < le16_to_cpu(bg->bg_list.l_next_free_rec); i++) {
                rec = &bg->bg_list.l_recs[i];
                if (ocfs2_bg_discontig_fix_by_rec(res, rec, cl)) {
                        res->sr_bg_blkno = bg_blkno;  /* Restore */
                        break;
                }
        }
}

static int ocfs2_search_one_group(struct ocfs2_alloc_context *ac,
                                  handle_t *handle,
                                  u32 bits_wanted,
                                  u32 min_bits,
                                  struct ocfs2_suballoc_result *res,
                                  u16 *bits_left, int *released)
{
        int ret;
        struct buffer_head *group_bh = NULL;
        struct ocfs2_group_desc *gd;
        struct ocfs2_dinode *di = (struct ocfs2_dinode *)ac->ac_bh->b_data;
        struct inode *alloc_inode = ac->ac_inode;

        ret = ocfs2_read_hint_group_descriptor(alloc_inode, di,
                                res->sr_bg_blkno, &group_bh, released);
        if (*released) {
                return 0;
        } else if (ret < 0) {
                mlog_errno(ret);
                return ret;
        }

        gd = (struct ocfs2_group_desc *) group_bh->b_data;
        ret = ac->ac_group_search(alloc_inode, group_bh, bits_wanted, min_bits,
                                  ac->ac_max_block, res);
        if (ret < 0) {
                if (ret != -ENOSPC)
                        mlog_errno(ret);
                goto out;
        }

        if (!ret)
                ocfs2_bg_discontig_fix_result(ac, gd, res);

        /*
         * sr_bg_blkno might have been changed by
         * ocfs2_bg_discontig_fix_result
         */
        res->sr_bg_stable_blkno = group_bh->b_blocknr;

        if (ac->ac_find_loc_only)
                goto out_loc_only;

        ret = ocfs2_alloc_dinode_update_counts(alloc_inode, handle, ac->ac_bh,
                                               res->sr_bits,
                                               le16_to_cpu(gd->bg_chain));
        if (ret < 0) {
                mlog_errno(ret);
                goto out;
        }

        ret = ocfs2_block_group_set_bits(handle, alloc_inode, gd, group_bh,
                                         res->sr_bit_offset, res->sr_bits,
                                         res->sr_max_contig_bits, 0);
        if (ret < 0) {
                ocfs2_rollback_alloc_dinode_counts(alloc_inode, ac->ac_bh,
                                               res->sr_bits,
                                               le16_to_cpu(gd->bg_chain));
                mlog_errno(ret);
        }

out_loc_only:
        *bits_left = le16_to_cpu(gd->bg_free_bits_count);

out:
        brelse(group_bh);

        return ret;
}

static int ocfs2_search_chain(struct ocfs2_alloc_context *ac,
                              handle_t *handle,
                              u32 bits_wanted,
                              u32 min_bits,
                              struct ocfs2_suballoc_result *res,
                              u16 *bits_left)
{
        int status;
        u16 chain;
        u32 contig_bits;
        u64 next_group;
        struct inode *alloc_inode = ac->ac_inode;
        struct buffer_head *group_bh = NULL;
        struct buffer_head *prev_group_bh = NULL;
        struct ocfs2_dinode *fe = (struct ocfs2_dinode *) ac->ac_bh->b_data;
        struct ocfs2_chain_list *cl = (struct ocfs2_chain_list *) &fe->id2.i_chain;
        struct ocfs2_group_desc *bg;

        chain = ac->ac_chain;
        trace_ocfs2_search_chain_begin(
                (unsigned long long)OCFS2_I(alloc_inode)->ip_blkno,
                bits_wanted, chain);

        status = ocfs2_read_group_descriptor(alloc_inode, fe,
                                             le64_to_cpu(cl->cl_recs[chain].c_blkno),
                                             &group_bh);
        if (status < 0) {
                mlog_errno(status);
                goto bail;
        }
        bg = (struct ocfs2_group_desc *) group_bh->b_data;

        status = -ENOSPC;
        /* for now, the chain search is a bit simplistic. We just use
         * the 1st group with any empty bits. */
        while (1) {
                if (ac->ac_which == OCFS2_AC_USE_MAIN_DISCONTIG) {
                        contig_bits = le16_to_cpu(bg->bg_contig_free_bits);
                        if (!contig_bits)
                                contig_bits = ocfs2_find_max_contig_free_bits(bg->bg_bitmap,
                                                le16_to_cpu(bg->bg_bits), 0);
                        if (bits_wanted > contig_bits && contig_bits >= min_bits)
                                bits_wanted = contig_bits;
                }

                status = ac->ac_group_search(alloc_inode, group_bh,
                                bits_wanted, min_bits,
                                ac->ac_max_block, res);
                if (status != -ENOSPC)
                        break;
                if (!bg->bg_next_group)
                        break;

                brelse(prev_group_bh);
                prev_group_bh = NULL;

                next_group = le64_to_cpu(bg->bg_next_group);
                prev_group_bh = group_bh;
                group_bh = NULL;
                status = ocfs2_read_group_descriptor(alloc_inode, fe,
                                                     next_group, &group_bh);
                if (status < 0) {
                        mlog_errno(status);
                        goto bail;
                }
                bg = (struct ocfs2_group_desc *) group_bh->b_data;
        }
        if (status < 0) {
                if (status != -ENOSPC)
                        mlog_errno(status);
                goto bail;
        }

        trace_ocfs2_search_chain_succ(
                (unsigned long long)le64_to_cpu(bg->bg_blkno), res->sr_bits);

        res->sr_bg_blkno = le64_to_cpu(bg->bg_blkno);

        BUG_ON(res->sr_bits == 0);
        if (!status)
                ocfs2_bg_discontig_fix_result(ac, bg, res);

        /*
         * sr_bg_blkno might have been changed by
         * ocfs2_bg_discontig_fix_result
         */
        res->sr_bg_stable_blkno = group_bh->b_blocknr;

        /*
         * Keep track of previous block descriptor read. When
         * we find a target, if we have read more than X
         * number of descriptors, and the target is reasonably
         * empty, relink him to top of his chain.
         *
         * We've read 0 extra blocks and only send one more to
         * the transaction, yet the next guy to search has a
         * much easier time.
         *
         * Do this *after* figuring out how many bits we're taking out
         * of our target group.
         */
        if (!ac->ac_disable_chain_relink &&
            (prev_group_bh) &&
            (ocfs2_block_group_reasonably_empty(bg, res->sr_bits))) {
                status = ocfs2_relink_block_group(handle, alloc_inode,
                                                  ac->ac_bh, group_bh,
                                                  prev_group_bh, chain);
                if (status < 0) {
                        mlog_errno(status);
                        goto bail;
                }
        }

        if (ac->ac_find_loc_only)
                goto out_loc_only;

        status = ocfs2_alloc_dinode_update_counts(alloc_inode, handle,
                                                  ac->ac_bh, res->sr_bits,
                                                  chain);
        if (status) {
                mlog_errno(status);
                goto bail;
        }

        status = ocfs2_block_group_set_bits(handle,
                                            alloc_inode,
                                            bg,
                                            group_bh,
                                            res->sr_bit_offset,
                                            res->sr_bits,
                                            res->sr_max_contig_bits,
                                            0);
        if (status < 0) {
                ocfs2_rollback_alloc_dinode_counts(alloc_inode,
                                        ac->ac_bh, res->sr_bits, chain);
                mlog_errno(status);
                goto bail;
        }

        trace_ocfs2_search_chain_end(
                        (unsigned long long)le64_to_cpu(fe->i_blkno),
                        res->sr_bits);

out_loc_only:
        *bits_left = le16_to_cpu(bg->bg_free_bits_count);
bail:
        brelse(group_bh);
        brelse(prev_group_bh);

        if (status)
                mlog_errno(status);
        return status;
}

/* will give out up to bits_wanted contiguous bits. */
static int ocfs2_claim_suballoc_bits(struct ocfs2_alloc_context *ac,
                                     handle_t *handle,
                                     u32 bits_wanted,
                                     u32 min_bits,
                                     struct ocfs2_suballoc_result *res)
{
        int status;
        int released = 0;
        u16 victim, i;
        u16 bits_left = 0;
        u64 hint = ac->ac_last_group;
        struct ocfs2_chain_list *cl;
        struct ocfs2_dinode *fe;

        BUG_ON(ac->ac_bits_given >= ac->ac_bits_wanted);
        BUG_ON(bits_wanted > (ac->ac_bits_wanted - ac->ac_bits_given));
        BUG_ON(!ac->ac_bh);

        fe = (struct ocfs2_dinode *) ac->ac_bh->b_data;

        /* The bh was validated by the inode read during
         * ocfs2_reserve_suballoc_bits().  Any corruption is a code bug. */
        BUG_ON(!OCFS2_IS_VALID_DINODE(fe));

        if (le32_to_cpu(fe->id1.bitmap1.i_used) >=
            le32_to_cpu(fe->id1.bitmap1.i_total)) {
                status = ocfs2_error(ac->ac_inode->i_sb,
                                     "Chain allocator dinode %llu has %u used bits but only %u total\n",
                                     (unsigned long long)le64_to_cpu(fe->i_blkno),
                                     le32_to_cpu(fe->id1.bitmap1.i_used),
                                     le32_to_cpu(fe->id1.bitmap1.i_total));
                goto bail;
        }

        /* the hint bg may already be released, we quiet search this group. */
        res->sr_bg_blkno = hint;
        if (res->sr_bg_blkno) {
                /* Attempt to short-circuit the usual search mechanism
                 * by jumping straight to the most recently used
                 * allocation group. This helps us maintain some
                 * contiguousness across allocations. */
                status = ocfs2_search_one_group(ac, handle, bits_wanted,
                                                min_bits, res, &bits_left,
                                                &released);
                if (released) {
                        res->sr_bg_blkno = 0;
                        goto chain_search;
                }
                if (!status)
                        goto set_hint;
                if (status < 0 && status != -ENOSPC) {
                        mlog_errno(status);
                        goto bail;
                }
        }
chain_search:
        cl = (struct ocfs2_chain_list *) &fe->id2.i_chain;
        if (!le16_to_cpu(cl->cl_next_free_rec) ||
            le16_to_cpu(cl->cl_next_free_rec) > le16_to_cpu(cl->cl_count)) {
                status = ocfs2_error(ac->ac_inode->i_sb,
                                     "Chain allocator dinode %llu has invalid next "
                                     "free chain record %u, but only %u total\n",
                                     (unsigned long long)le64_to_cpu(fe->i_blkno),
                                     le16_to_cpu(cl->cl_next_free_rec),
                                     le16_to_cpu(cl->cl_count));
                goto bail;
        }

        victim = ocfs2_find_victim_chain(cl);
        ac->ac_chain = victim;

search:
        status = ocfs2_search_chain(ac, handle, bits_wanted, min_bits,
                                    res, &bits_left);
        if (!status) {
                if (ocfs2_is_cluster_bitmap(ac->ac_inode))
                        hint = res->sr_bg_blkno;
                else
                        hint = ocfs2_group_from_res(res);
                goto set_hint;
        }
        if (status < 0 && status != -ENOSPC) {
                mlog_errno(status);
                goto bail;
        }

        trace_ocfs2_claim_suballoc_bits(victim);

        /* If we didn't pick a good victim, then just default to
         * searching each chain in order. Don't allow chain relinking
         * because we only calculate enough journal credits for one
         * relink per alloc. */
        ac->ac_disable_chain_relink = 1;
        for (i = 0; i < le16_to_cpu(cl->cl_next_free_rec); i ++) {
                if (i == victim)
                        continue;
                if (le32_to_cpu(cl->cl_recs[i].c_free) < bits_wanted)
                        continue;

                ac->ac_chain = i;
                status = ocfs2_search_chain(ac, handle, bits_wanted, min_bits,
                                            res, &bits_left);
                if (!status) {
                        hint = ocfs2_group_from_res(res);
                        break;
                }
                if (status < 0 && status != -ENOSPC) {
                        mlog_errno(status);
                        goto bail;
                }
        }

        /* Chains can't supply the bits_wanted contiguous space.
         * We should switch to using every single bit when allocating
         * from the global bitmap. */
        if (i == le16_to_cpu(cl->cl_next_free_rec) &&
            status == -ENOSPC && ac->ac_which == OCFS2_AC_USE_MAIN) {
                ac->ac_which = OCFS2_AC_USE_MAIN_DISCONTIG;
                ac->ac_chain = victim;
                goto search;
        }

set_hint:
        if (status != -ENOSPC) {
                /* If the next search of this group is not likely to
                 * yield a suitable extent, then we reset the last
                 * group hint so as to not waste a disk read */
                if (bits_left < min_bits)
                        ac->ac_last_group = 0;
                else
                        ac->ac_last_group = hint;
        }

bail:
        if (status)
                mlog_errno(status);
        return status;
}

int ocfs2_claim_metadata(handle_t *handle,
                         struct ocfs2_alloc_context *ac,
                         u32 bits_wanted,
                         u64 *suballoc_loc,
                         u16 *suballoc_bit_start,
                         unsigned int *num_bits,
                         u64 *blkno_start)
{
        int status;
        struct ocfs2_suballoc_result res = { .sr_blkno = 0, };

        BUG_ON(!ac);
        BUG_ON(ac->ac_bits_wanted < (ac->ac_bits_given + bits_wanted));
        BUG_ON(ac->ac_which != OCFS2_AC_USE_META);

        status = ocfs2_claim_suballoc_bits(ac,
                                           handle,
                                           bits_wanted,
                                           1,
                                           &res);
        if (status < 0) {
                mlog_errno(status);
                goto bail;
        }
        atomic_inc(&OCFS2_SB(ac->ac_inode->i_sb)->alloc_stats.bg_allocs);

        *suballoc_loc = res.sr_bg_blkno;
        *suballoc_bit_start = res.sr_bit_offset;
        *blkno_start = res.sr_blkno;
        ac->ac_bits_given += res.sr_bits;
        *num_bits = res.sr_bits;
        status = 0;
bail:
        if (status)
                mlog_errno(status);
        return status;
}

/*
 * after ocfs2 has the ability to release block group unused space,
 * the ->ip_last_used_group may be invalid. so this function returns
 * ac->ac_last_group need to verify.
 * refer the 'hint' in ocfs2_claim_suballoc_bits() for more details.
 */
static void ocfs2_init_inode_ac_group(struct inode *dir,
                                      struct buffer_head *parent_di_bh,
                                      struct ocfs2_alloc_context *ac)
{
        struct ocfs2_dinode *di = (struct ocfs2_dinode *)parent_di_bh->b_data;
        /*
         * Try to allocate inodes from some specific group.
         *
         * If the parent dir has recorded the last group used in allocation,
         * cool, use it. Otherwise if we try to allocate new inode from the
         * same slot the parent dir belongs to, use the same chunk.
         *
         * We are very careful here to avoid the mistake of setting
         * ac_last_group to a group descriptor from a different (unlocked) slot.
         */
        if (OCFS2_I(dir)->ip_last_used_group &&
            OCFS2_I(dir)->ip_last_used_slot == ac->ac_alloc_slot)
                ac->ac_last_group = OCFS2_I(dir)->ip_last_used_group;
        else if (le16_to_cpu(di->i_suballoc_slot) == ac->ac_alloc_slot) {
                if (di->i_suballoc_loc)
                        ac->ac_last_group = le64_to_cpu(di->i_suballoc_loc);
                else
                        ac->ac_last_group = ocfs2_which_suballoc_group(
                                        le64_to_cpu(di->i_blkno),
                                        le16_to_cpu(di->i_suballoc_bit));
        }
}

static inline void ocfs2_save_inode_ac_group(struct inode *dir,
                                             struct ocfs2_alloc_context *ac)
{
        OCFS2_I(dir)->ip_last_used_group = ac->ac_last_group;
        OCFS2_I(dir)->ip_last_used_slot = ac->ac_alloc_slot;
}

int ocfs2_find_new_inode_loc(struct inode *dir,
                             struct buffer_head *parent_fe_bh,
                             struct ocfs2_alloc_context *ac,
                             u64 *fe_blkno)
{
        int ret;
        handle_t *handle = NULL;
        struct ocfs2_suballoc_result *res;

        BUG_ON(!ac);
        BUG_ON(ac->ac_bits_given != 0);
        BUG_ON(ac->ac_bits_wanted != 1);
        BUG_ON(ac->ac_which != OCFS2_AC_USE_INODE);

        res = kzalloc_obj(*res, GFP_NOFS);
        if (res == NULL) {
                ret = -ENOMEM;
                mlog_errno(ret);
                goto out;
        }

        ocfs2_init_inode_ac_group(dir, parent_fe_bh, ac);

        /*
         * The handle started here is for chain relink. Alternatively,
         * we could just disable relink for these calls.
         */
        handle = ocfs2_start_trans(OCFS2_SB(dir->i_sb), OCFS2_SUBALLOC_ALLOC);
        if (IS_ERR(handle)) {
                ret = PTR_ERR(handle);
                handle = NULL;
                mlog_errno(ret);
                goto out;
        }

        /*
         * This will instruct ocfs2_claim_suballoc_bits and
         * ocfs2_search_one_group to search but save actual allocation
         * for later.
         */
        ac->ac_find_loc_only = 1;

        ret = ocfs2_claim_suballoc_bits(ac, handle, 1, 1, res);
        if (ret < 0) {
                mlog_errno(ret);
                goto out;
        }

        ac->ac_find_loc_priv = res;
        *fe_blkno = res->sr_blkno;
        ocfs2_update_inode_fsync_trans(handle, dir, 0);
out:
        if (handle)
                ocfs2_commit_trans(OCFS2_SB(dir->i_sb), handle);

        if (ret)
                kfree(res);

        return ret;
}

int ocfs2_claim_new_inode_at_loc(handle_t *handle,
                                 struct inode *dir,
                                 struct ocfs2_alloc_context *ac,
                                 u64 *suballoc_loc,
                                 u16 *suballoc_bit,
                                 u64 di_blkno)
{
        int ret;
        u16 chain;
        struct ocfs2_suballoc_result *res = ac->ac_find_loc_priv;
        struct buffer_head *bg_bh = NULL;
        struct ocfs2_group_desc *bg;
        struct ocfs2_dinode *di = (struct ocfs2_dinode *) ac->ac_bh->b_data;

        /*
         * Since di_blkno is being passed back in, we check for any
         * inconsistencies which may have happened between
         * calls. These are code bugs as di_blkno is not expected to
         * change once returned from ocfs2_find_new_inode_loc()
         */
        BUG_ON(res->sr_blkno != di_blkno);

        ret = ocfs2_read_group_descriptor(ac->ac_inode, di,
                                          res->sr_bg_stable_blkno, &bg_bh);
        if (ret) {
                mlog_errno(ret);
                goto out;
        }

        bg = (struct ocfs2_group_desc *) bg_bh->b_data;
        chain = le16_to_cpu(bg->bg_chain);

        ret = ocfs2_alloc_dinode_update_counts(ac->ac_inode, handle,
                                               ac->ac_bh, res->sr_bits,
                                               chain);
        if (ret) {
                mlog_errno(ret);
                goto out;
        }

        ret = ocfs2_block_group_set_bits(handle,
                                         ac->ac_inode,
                                         bg,
                                         bg_bh,
                                         res->sr_bit_offset,
                                         res->sr_bits,
                                         res->sr_max_contig_bits,
                                         0);
        if (ret < 0) {
                ocfs2_rollback_alloc_dinode_counts(ac->ac_inode,
                                               ac->ac_bh, res->sr_bits, chain);
                mlog_errno(ret);
                goto out;
        }

        trace_ocfs2_claim_new_inode_at_loc((unsigned long long)di_blkno,
                                           res->sr_bits);

        atomic_inc(&OCFS2_SB(ac->ac_inode->i_sb)->alloc_stats.bg_allocs);

        BUG_ON(res->sr_bits != 1);

        *suballoc_loc = res->sr_bg_blkno;
        *suballoc_bit = res->sr_bit_offset;
        ac->ac_bits_given++;
        ocfs2_save_inode_ac_group(dir, ac);

out:
        brelse(bg_bh);

        return ret;
}

int ocfs2_claim_new_inode(handle_t *handle,
                          struct inode *dir,
                          struct buffer_head *parent_fe_bh,
                          struct ocfs2_alloc_context *ac,
                          u64 *suballoc_loc,
                          u16 *suballoc_bit,
                          u64 *fe_blkno)
{
        int status;
        struct ocfs2_suballoc_result res;

        BUG_ON(!ac);
        BUG_ON(ac->ac_bits_given != 0);
        BUG_ON(ac->ac_bits_wanted != 1);
        BUG_ON(ac->ac_which != OCFS2_AC_USE_INODE);

        ocfs2_init_inode_ac_group(dir, parent_fe_bh, ac);

        status = ocfs2_claim_suballoc_bits(ac,
                                           handle,
                                           1,
                                           1,
                                           &res);
        if (status < 0) {
                mlog_errno(status);
                goto bail;
        }
        atomic_inc(&OCFS2_SB(ac->ac_inode->i_sb)->alloc_stats.bg_allocs);

        BUG_ON(res.sr_bits != 1);

        *suballoc_loc = res.sr_bg_blkno;
        *suballoc_bit = res.sr_bit_offset;
        *fe_blkno = res.sr_blkno;
        ac->ac_bits_given++;
        ocfs2_save_inode_ac_group(dir, ac);
        status = 0;
bail:
        if (status)
                mlog_errno(status);
        return status;
}

/* translate a group desc. blkno and it's bitmap offset into
 * disk cluster offset. */
static inline u32 ocfs2_desc_bitmap_to_cluster_off(struct inode *inode,
                                                   u64 bg_blkno,
                                                   u16 bg_bit_off)
{
        struct ocfs2_super *osb = OCFS2_SB(inode->i_sb);
        u32 cluster = 0;

        BUG_ON(!ocfs2_is_cluster_bitmap(inode));

        if (bg_blkno != osb->first_cluster_group_blkno)
                cluster = ocfs2_blocks_to_clusters(inode->i_sb, bg_blkno);
        cluster += (u32) bg_bit_off;
        return cluster;
}

/* given a cluster offset, calculate which block group it belongs to
 * and return that block offset. */
u64 ocfs2_which_cluster_group(struct inode *inode, u32 cluster)
{
        struct ocfs2_super *osb = OCFS2_SB(inode->i_sb);
        u32 group_no;

        BUG_ON(!ocfs2_is_cluster_bitmap(inode));

        group_no = cluster / osb->bitmap_cpg;
        if (!group_no)
                return osb->first_cluster_group_blkno;
        return ocfs2_clusters_to_blocks(inode->i_sb,
                                        group_no * osb->bitmap_cpg);
}

/* given the block number of a cluster start, calculate which cluster
 * group and descriptor bitmap offset that corresponds to. */
static inline void ocfs2_block_to_cluster_group(struct inode *inode,
                                                u64 data_blkno,
                                                u64 *bg_blkno,
                                                u16 *bg_bit_off)
{
        struct ocfs2_super *osb = OCFS2_SB(inode->i_sb);
        u32 data_cluster = ocfs2_blocks_to_clusters(osb->sb, data_blkno);

        BUG_ON(!ocfs2_is_cluster_bitmap(inode));

        *bg_blkno = ocfs2_which_cluster_group(inode,
                                              data_cluster);

        if (*bg_blkno == osb->first_cluster_group_blkno)
                *bg_bit_off = (u16) data_cluster;
        else
                *bg_bit_off = (u16) ocfs2_blocks_to_clusters(osb->sb,
                                                             data_blkno - *bg_blkno);
}

/*
 * min_bits - minimum contiguous chunk from this total allocation we
 * can handle. set to what we asked for originally for a full
 * contig. allocation, set to '1' to indicate we can deal with extents
 * of any size.
 */
int __ocfs2_claim_clusters(handle_t *handle,
                           struct ocfs2_alloc_context *ac,
                           u32 min_clusters,
                           u32 max_clusters,
                           u32 *cluster_start,
                           u32 *num_clusters)
{
        int status;
        unsigned int bits_wanted = max_clusters;
        struct ocfs2_suballoc_result res = { .sr_blkno = 0, };
        struct ocfs2_super *osb = OCFS2_SB(ac->ac_inode->i_sb);

        BUG_ON(ac->ac_bits_given >= ac->ac_bits_wanted);

        BUG_ON(ac->ac_which != OCFS2_AC_USE_LOCAL
               && ac->ac_which != OCFS2_AC_USE_MAIN
               && ac->ac_which != OCFS2_AC_USE_MAIN_DISCONTIG);

        if (ac->ac_which == OCFS2_AC_USE_LOCAL) {
                WARN_ON(min_clusters > 1);

                status = ocfs2_claim_local_alloc_bits(osb,
                                                      handle,
                                                      ac,
                                                      bits_wanted,
                                                      cluster_start,
                                                      num_clusters);
                if (!status)
                        atomic_inc(&osb->alloc_stats.local_data);
        } else {
                if (min_clusters > (osb->bitmap_cpg - 1)) {
                        /* The only paths asking for contiguousness
                         * should know about this already. */
                        mlog(ML_ERROR, "minimum allocation requested %u exceeds "
                             "group bitmap size %u!\n", min_clusters,
                             osb->bitmap_cpg);
                        status = -ENOSPC;
                        goto bail;
                }
                /* clamp the current request down to a realistic size. */
                if (bits_wanted > (osb->bitmap_cpg - 1))
                        bits_wanted = osb->bitmap_cpg - 1;

                status = ocfs2_claim_suballoc_bits(ac,
                                                   handle,
                                                   bits_wanted,
                                                   min_clusters,
                                                   &res);
                if (!status) {
                        BUG_ON(res.sr_blkno); /* cluster alloc can't set */
                        *cluster_start =
                                ocfs2_desc_bitmap_to_cluster_off(ac->ac_inode,
                                                                 res.sr_bg_blkno,
                                                                 res.sr_bit_offset);
                        atomic_inc(&osb->alloc_stats.bitmap_data);
                        *num_clusters = res.sr_bits;
                }
        }
        if (status < 0) {
                if (status != -ENOSPC)
                        mlog_errno(status);
                goto bail;
        }

        ac->ac_bits_given += *num_clusters;

bail:
        if (status)
                mlog_errno(status);
        return status;
}

int ocfs2_claim_clusters(handle_t *handle,
                         struct ocfs2_alloc_context *ac,
                         u32 min_clusters,
                         u32 *cluster_start,
                         u32 *num_clusters)
{
        unsigned int bits_wanted = ac->ac_bits_wanted - ac->ac_bits_given;

        return __ocfs2_claim_clusters(handle, ac, min_clusters,
                                      bits_wanted, cluster_start, num_clusters);
}

static int ocfs2_block_group_clear_bits(handle_t *handle,
                                        struct inode *alloc_inode,
                                        struct ocfs2_group_desc *bg,
                                        struct buffer_head *group_bh,
                                        unsigned int bit_off,
                                        unsigned int num_bits,
                                        unsigned int max_contig_bits,
                                        void (*undo_fn)(unsigned int bit,
                                                        unsigned long *bmap))
{
        int status;
        unsigned int tmp;
        u16 contig_bits;
        struct ocfs2_group_desc *undo_bg = NULL;
        struct journal_head *jh;

        /* The caller got this descriptor from
         * ocfs2_read_group_descriptor().  Any corruption is a code bug. */
        BUG_ON(!OCFS2_IS_VALID_GROUP_DESC(bg));

        trace_ocfs2_block_group_clear_bits(bit_off, num_bits);

        BUG_ON(undo_fn && !ocfs2_is_cluster_bitmap(alloc_inode));
        status = ocfs2_journal_access_gd(handle, INODE_CACHE(alloc_inode),
                                         group_bh,
                                         undo_fn ?
                                         OCFS2_JOURNAL_ACCESS_UNDO :
                                         OCFS2_JOURNAL_ACCESS_WRITE);
        if (status < 0) {
                mlog_errno(status);
                goto bail;
        }

        jh = bh2jh(group_bh);
        if (undo_fn) {
                spin_lock(&jh->b_state_lock);
                undo_bg = (struct ocfs2_group_desc *) jh->b_committed_data;
                BUG_ON(!undo_bg);
        }

        tmp = num_bits;
        while(tmp--) {
                ocfs2_clear_bit((bit_off + tmp),
                                (unsigned long *) bg->bg_bitmap);
                if (undo_fn)
                        undo_fn(bit_off + tmp,
                                (unsigned long *) undo_bg->bg_bitmap);
        }
        le16_add_cpu(&bg->bg_free_bits_count, num_bits);
        if (le16_to_cpu(bg->bg_free_bits_count) > le16_to_cpu(bg->bg_bits)) {
                if (undo_fn)
                        spin_unlock(&jh->b_state_lock);
                return ocfs2_error(alloc_inode->i_sb, "Group descriptor # %llu has bit count %u but claims %u are freed. num_bits %d\n",
                                   (unsigned long long)le64_to_cpu(bg->bg_blkno),
                                   le16_to_cpu(bg->bg_bits),
                                   le16_to_cpu(bg->bg_free_bits_count),
                                   num_bits);
        }

        /*
         * TODO: even 'num_bits == 1' (the worst case, release 1 cluster),
         * we still need to rescan whole bitmap.
         */
        if (ocfs2_is_cluster_bitmap(alloc_inode)) {
                contig_bits = ocfs2_find_max_contig_free_bits(bg->bg_bitmap,
                                    le16_to_cpu(bg->bg_bits), 0);
                if (contig_bits > max_contig_bits)
                        max_contig_bits = contig_bits;
                bg->bg_contig_free_bits = cpu_to_le16(max_contig_bits);
        } else {
                bg->bg_contig_free_bits = 0;
        }

        if (undo_fn)
                spin_unlock(&jh->b_state_lock);

        ocfs2_journal_dirty(handle, group_bh);
bail:
        return status;
}

/*
 * Reclaim the suballocator managed space to main bitmap.
 * This function first works on the suballocator to perform the
 * cleanup rec/alloc_inode job, then switches to the main bitmap
 * to reclaim released space.
 *
 * handle: The transaction handle
 * alloc_inode: The suballoc inode
 * alloc_bh: The buffer_head of suballoc inode
 * group_bh: The group descriptor buffer_head of suballocator managed.
 *           Caller should release the input group_bh.
 */
static int _ocfs2_reclaim_suballoc_to_main(handle_t *handle,
                        struct inode *alloc_inode,
                        struct buffer_head *alloc_bh,
                        struct buffer_head *group_bh)
{
        int idx, status = 0;
        int i, next_free_rec, len = 0;
        __le16 old_bg_contig_free_bits = 0;
        u16 start_bit;
        u32 tmp_used;
        u64 bg_blkno, start_blk;
        unsigned int count;
        struct ocfs2_chain_rec *rec;
        struct buffer_head *main_bm_bh = NULL;
        struct inode *main_bm_inode = NULL;
        struct ocfs2_super *osb = OCFS2_SB(alloc_inode->i_sb);
        struct ocfs2_dinode *fe = (struct ocfs2_dinode *) alloc_bh->b_data;
        struct ocfs2_chain_list *cl = &fe->id2.i_chain;
        struct ocfs2_group_desc *group = (struct ocfs2_group_desc *) group_bh->b_data;

        idx = le16_to_cpu(group->bg_chain);
        rec = &(cl->cl_recs[idx]);

        status = ocfs2_extend_trans(handle,
                                ocfs2_calc_group_alloc_credits(osb->sb,
                                                 le16_to_cpu(cl->cl_cpg)));
        if (status) {
                mlog_errno(status);
                goto bail;
        }
        status = ocfs2_journal_access_di(handle, INODE_CACHE(alloc_inode),
                                         alloc_bh, OCFS2_JOURNAL_ACCESS_WRITE);
        if (status < 0) {
                mlog_errno(status);
                goto bail;
        }

        /*
         * Only clear the suballocator rec item in-place.
         *
         * If idx is not the last, we don't compress (remove the empty item)
         * the cl_recs[]. If not, we need to do lots jobs.
         *
         * Compress cl_recs[] code example:
         * if (idx != cl->cl_next_free_rec - 1)
         *     memmove(&cl->cl_recs[idx], &cl->cl_recs[idx + 1],
         *         sizeof(struct ocfs2_chain_rec) *
         *         (cl->cl_next_free_rec - idx - 1));
         * for(i = idx; i < cl->cl_next_free_rec-1; i++) {
         *     group->bg_chain = "later group->bg_chain";
         *     group->bg_blkno = xxx;
         *     ... ...
         * }
         */

        tmp_used = le32_to_cpu(fe->id1.bitmap1.i_total);
        fe->id1.bitmap1.i_total = cpu_to_le32(tmp_used - le32_to_cpu(rec->c_total));

        /* Substraction 1 for the block group itself */
        tmp_used = le32_to_cpu(fe->id1.bitmap1.i_used);
        fe->id1.bitmap1.i_used = cpu_to_le32(tmp_used - 1);

        tmp_used = le32_to_cpu(fe->i_clusters);
        fe->i_clusters = cpu_to_le32(tmp_used - le16_to_cpu(cl->cl_cpg));

        spin_lock(&OCFS2_I(alloc_inode)->ip_lock);
        OCFS2_I(alloc_inode)->ip_clusters = le32_to_cpu(fe->i_clusters);
        fe->i_size = cpu_to_le64(ocfs2_clusters_to_bytes(alloc_inode->i_sb,
                                             le32_to_cpu(fe->i_clusters)));
        spin_unlock(&OCFS2_I(alloc_inode)->ip_lock);
        i_size_write(alloc_inode, le64_to_cpu(fe->i_size));
        alloc_inode->i_blocks = ocfs2_inode_sector_count(alloc_inode);

        ocfs2_journal_dirty(handle, alloc_bh);
        ocfs2_update_inode_fsync_trans(handle, alloc_inode, 0);

        start_blk = le64_to_cpu(rec->c_blkno);
        count = le32_to_cpu(rec->c_total) / le16_to_cpu(cl->cl_bpc);

        /*
         * If the rec is the last one, let's compress the chain list by
         * removing the empty cl_recs[] at the end.
         */
        next_free_rec = le16_to_cpu(cl->cl_next_free_rec);
        if (idx == (next_free_rec - 1)) {
                len++; /* the last item should be counted first */
                for (i = (next_free_rec - 2); i > 0; i--) {
                        if (cl->cl_recs[i].c_free == cl->cl_recs[i].c_total)
                                len++;
                        else
                                break;
                }
        }
        le16_add_cpu(&cl->cl_next_free_rec, -len);

        rec->c_free = 0;
        rec->c_total = 0;
        rec->c_blkno = 0;
        ocfs2_remove_from_cache(INODE_CACHE(alloc_inode), group_bh);
        memset(group, 0, sizeof(struct ocfs2_group_desc));

        /* prepare job for reclaim clusters */
        main_bm_inode = ocfs2_get_system_file_inode(osb,
                                                    GLOBAL_BITMAP_SYSTEM_INODE,
                                                    OCFS2_INVALID_SLOT);
        if (!main_bm_inode)
                goto bail; /* ignore the error in reclaim path */

        inode_lock(main_bm_inode);

        status = ocfs2_inode_lock(main_bm_inode, &main_bm_bh, 1);
        if (status < 0)
                goto free_bm_inode; /* ignore the error in reclaim path */

        ocfs2_block_to_cluster_group(main_bm_inode, start_blk, &bg_blkno,
                                     &start_bit);
        fe = (struct ocfs2_dinode *) main_bm_bh->b_data;
        cl = &fe->id2.i_chain;
        /* reuse group_bh, caller will release the input group_bh */
        group_bh = NULL;

        /* reclaim clusters to global_bitmap */
        status = ocfs2_read_group_descriptor(main_bm_inode, fe, bg_blkno,
                                             &group_bh);
        if (status < 0) {
                mlog_errno(status);
                goto free_bm_bh;
        }
        group = (struct ocfs2_group_desc *) group_bh->b_data;

        if ((count + start_bit) > le16_to_cpu(group->bg_bits)) {
                ocfs2_error(alloc_inode->i_sb,
                        "reclaim length (%d) beyands block group length (%d)",
                        count + start_bit, le16_to_cpu(group->bg_bits));
                goto free_group_bh;
        }

        old_bg_contig_free_bits = group->bg_contig_free_bits;
        status = ocfs2_block_group_clear_bits(handle, main_bm_inode,
                                              group, group_bh,
                                              start_bit, count, 0,
                                              _ocfs2_clear_bit);
        if (status < 0) {
                mlog_errno(status);
                goto free_group_bh;
        }

        status = ocfs2_journal_access_di(handle, INODE_CACHE(main_bm_inode),
                                         main_bm_bh, OCFS2_JOURNAL_ACCESS_WRITE);
        if (status < 0) {
                mlog_errno(status);
                ocfs2_block_group_set_bits(handle, main_bm_inode, group, group_bh,
                                start_bit, count,
                                le16_to_cpu(old_bg_contig_free_bits), 1);
                goto free_group_bh;
        }

        idx = le16_to_cpu(group->bg_chain);
        rec = &(cl->cl_recs[idx]);

        le32_add_cpu(&rec->c_free, count);
        tmp_used = le32_to_cpu(fe->id1.bitmap1.i_used);
        fe->id1.bitmap1.i_used = cpu_to_le32(tmp_used - count);
        ocfs2_journal_dirty(handle, main_bm_bh);

free_group_bh:
        brelse(group_bh);

free_bm_bh:
        ocfs2_inode_unlock(main_bm_inode, 1);
        brelse(main_bm_bh);

free_bm_inode:
        inode_unlock(main_bm_inode);
        iput(main_bm_inode);

bail:
        return status;
}

/*
 * expects the suballoc inode to already be locked.
 */
static int _ocfs2_free_suballoc_bits(handle_t *handle,
                                     struct inode *alloc_inode,
                                     struct buffer_head *alloc_bh,
                                     unsigned int start_bit,
                                     u64 bg_blkno,
                                     unsigned int count,
                                     void (*undo_fn)(unsigned int bit,
                                                     unsigned long *bitmap))
{
        int idx, status = 0;
        u32 tmp_used;
        struct ocfs2_dinode *fe = (struct ocfs2_dinode *) alloc_bh->b_data;
        struct ocfs2_chain_list *cl = &fe->id2.i_chain;
        struct buffer_head *group_bh = NULL;
        struct ocfs2_group_desc *group;
        struct ocfs2_chain_rec *rec;
        __le16 old_bg_contig_free_bits = 0;

        /* The alloc_bh comes from ocfs2_free_dinode() or
         * ocfs2_free_clusters().  The callers have all locked the
         * allocator and gotten alloc_bh from the lock call.  This
         * validates the dinode buffer.  Any corruption that has happened
         * is a code bug. */
        BUG_ON(!OCFS2_IS_VALID_DINODE(fe));
        BUG_ON((count + start_bit) > ocfs2_bits_per_group(cl));

        trace_ocfs2_free_suballoc_bits(
                (unsigned long long)OCFS2_I(alloc_inode)->ip_blkno,
                (unsigned long long)bg_blkno,
                start_bit, count);

        status = ocfs2_read_group_descriptor(alloc_inode, fe, bg_blkno,
                                             &group_bh);
        if (status < 0) {
                mlog_errno(status);
                goto bail;
        }
        group = (struct ocfs2_group_desc *) group_bh->b_data;

        BUG_ON((count + start_bit) > le16_to_cpu(group->bg_bits));

        if (ocfs2_is_cluster_bitmap(alloc_inode))
                old_bg_contig_free_bits = group->bg_contig_free_bits;
        status = ocfs2_block_group_clear_bits(handle, alloc_inode,
                                              group, group_bh,
                                              start_bit, count, 0, undo_fn);
        if (status < 0) {
                mlog_errno(status);
                goto bail;
        }

        status = ocfs2_journal_access_di(handle, INODE_CACHE(alloc_inode),
                                         alloc_bh, OCFS2_JOURNAL_ACCESS_WRITE);
        if (status < 0) {
                mlog_errno(status);
                ocfs2_block_group_set_bits(handle, alloc_inode, group, group_bh,
                                start_bit, count,
                                le16_to_cpu(old_bg_contig_free_bits), 1);
                goto bail;
        }

        idx = le16_to_cpu(group->bg_chain);
        rec = &(cl->cl_recs[idx]);

        le32_add_cpu(&rec->c_free, count);
        tmp_used = le32_to_cpu(fe->id1.bitmap1.i_used);
        fe->id1.bitmap1.i_used = cpu_to_le32(tmp_used - count);
        ocfs2_journal_dirty(handle, alloc_bh);

        /*
         * Reclaim suballocator free space.
         * Bypass: global_bitmap, non empty rec, first rec in cl_recs[]
         */
        if (ocfs2_is_cluster_bitmap(alloc_inode) ||
            (le32_to_cpu(rec->c_free) != (le32_to_cpu(rec->c_total) - 1)) ||
            (le16_to_cpu(cl->cl_next_free_rec) == 1)) {
                goto bail;
        }

        _ocfs2_reclaim_suballoc_to_main(handle, alloc_inode, alloc_bh, group_bh);

bail:
        brelse(group_bh);
        return status;
}

int ocfs2_free_suballoc_bits(handle_t *handle,
                             struct inode *alloc_inode,
                             struct buffer_head *alloc_bh,
                             unsigned int start_bit,
                             u64 bg_blkno,
                             unsigned int count)
{
        return _ocfs2_free_suballoc_bits(handle, alloc_inode, alloc_bh,
                                         start_bit, bg_blkno, count, NULL);
}

int ocfs2_free_dinode(handle_t *handle,
                      struct inode *inode_alloc_inode,
                      struct buffer_head *inode_alloc_bh,
                      struct ocfs2_dinode *di)
{
        u64 blk = le64_to_cpu(di->i_blkno);
        u16 bit = le16_to_cpu(di->i_suballoc_bit);
        u64 bg_blkno = ocfs2_which_suballoc_group(blk, bit);

        if (di->i_suballoc_loc)
                bg_blkno = le64_to_cpu(di->i_suballoc_loc);
        return ocfs2_free_suballoc_bits(handle, inode_alloc_inode,
                                        inode_alloc_bh, bit, bg_blkno, 1);
}

static int _ocfs2_free_clusters(handle_t *handle,
                                struct inode *bitmap_inode,
                                struct buffer_head *bitmap_bh,
                                u64 start_blk,
                                unsigned int num_clusters,
                                void (*undo_fn)(unsigned int bit,
                                                unsigned long *bitmap))
{
        int status;
        u16 bg_start_bit;
        u64 bg_blkno;

        /* You can't ever have a contiguous set of clusters
         * bigger than a block group bitmap so we never have to worry
         * about looping on them.
         * This is expensive. We can safely remove once this stuff has
         * gotten tested really well. */
        BUG_ON(start_blk != ocfs2_clusters_to_blocks(bitmap_inode->i_sb,
                                ocfs2_blocks_to_clusters(bitmap_inode->i_sb,
                                                         start_blk)));


        ocfs2_block_to_cluster_group(bitmap_inode, start_blk, &bg_blkno,
                                     &bg_start_bit);

        trace_ocfs2_free_clusters((unsigned long long)bg_blkno,
                        (unsigned long long)start_blk,
                        bg_start_bit, num_clusters);

        status = _ocfs2_free_suballoc_bits(handle, bitmap_inode, bitmap_bh,
                                           bg_start_bit, bg_blkno,
                                           num_clusters, undo_fn);
        if (status < 0) {
                mlog_errno(status);
                goto out;
        }

        ocfs2_local_alloc_seen_free_bits(OCFS2_SB(bitmap_inode->i_sb),
                                         num_clusters);

out:
        return status;
}

int ocfs2_free_clusters(handle_t *handle,
                        struct inode *bitmap_inode,
                        struct buffer_head *bitmap_bh,
                        u64 start_blk,
                        unsigned int num_clusters)
{
        return _ocfs2_free_clusters(handle, bitmap_inode, bitmap_bh,
                                    start_blk, num_clusters,
                                    _ocfs2_set_bit);
}

/*
 * Give never-used clusters back to the global bitmap.  We don't need
 * to protect these bits in the undo buffer.
 */
int ocfs2_release_clusters(handle_t *handle,
                           struct inode *bitmap_inode,
                           struct buffer_head *bitmap_bh,
                           u64 start_blk,
                           unsigned int num_clusters)
{
        return _ocfs2_free_clusters(handle, bitmap_inode, bitmap_bh,
                                    start_blk, num_clusters,
                                    _ocfs2_clear_bit);
}

/*
 * For a given allocation, determine which allocators will need to be
 * accessed, and lock them, reserving the appropriate number of bits.
 *
 * Sparse file systems call this from ocfs2_write_begin_nolock()
 * and ocfs2_allocate_unwritten_extents().
 *
 * File systems which don't support holes call this from
 * ocfs2_extend_allocation().
 */
int ocfs2_lock_allocators(struct inode *inode,
                          struct ocfs2_extent_tree *et,
                          u32 clusters_to_add, u32 extents_to_split,
                          struct ocfs2_alloc_context **data_ac,
                          struct ocfs2_alloc_context **meta_ac)
{
        int ret = 0, num_free_extents;
        unsigned int max_recs_needed = clusters_to_add + 2 * extents_to_split;
        struct ocfs2_super *osb = OCFS2_SB(inode->i_sb);

        *meta_ac = NULL;
        if (data_ac)
                *data_ac = NULL;

        BUG_ON(clusters_to_add != 0 && data_ac == NULL);

        num_free_extents = ocfs2_num_free_extents(et);
        if (num_free_extents < 0) {
                ret = num_free_extents;
                mlog_errno(ret);
                goto out;
        }

        /*
         * Sparse allocation file systems need to be more conservative
         * with reserving room for expansion - the actual allocation
         * happens while we've got a journal handle open so re-taking
         * a cluster lock (because we ran out of room for another
         * extent) will violate ordering rules.
         *
         * Most of the time we'll only be seeing this 1 cluster at a time
         * anyway.
         *
         * Always lock for any unwritten extents - we might want to
         * add blocks during a split.
         */
        if (!num_free_extents ||
            (ocfs2_sparse_alloc(osb) && num_free_extents < max_recs_needed)) {
                ret = ocfs2_reserve_new_metadata(osb, et->et_root_el, meta_ac);
                if (ret < 0) {
                        if (ret != -ENOSPC)
                                mlog_errno(ret);
                        goto out;
                }
        }

        if (clusters_to_add == 0)
                goto out;

        ret = ocfs2_reserve_clusters(osb, clusters_to_add, data_ac);
        if (ret < 0) {
                if (ret != -ENOSPC)
                        mlog_errno(ret);
                goto out;
        }

out:
        if (ret) {
                if (*meta_ac) {
                        ocfs2_free_alloc_context(*meta_ac);
                        *meta_ac = NULL;
                }

                /*
                 * We cannot have an error and a non null *data_ac.
                 */
        }

        return ret;
}

/*
 * Read the inode specified by blkno to get suballoc_slot and
 * suballoc_bit.
 */
static int ocfs2_get_suballoc_slot_bit(struct ocfs2_super *osb, u64 blkno,
                                       u16 *suballoc_slot, u64 *group_blkno,
                                       u16 *suballoc_bit)
{
        int status;
        struct buffer_head *inode_bh = NULL;
        struct ocfs2_dinode *inode_fe;

        trace_ocfs2_get_suballoc_slot_bit((unsigned long long)blkno);

        /* dirty read disk */
        status = ocfs2_read_blocks_sync(osb, blkno, 1, &inode_bh);
        if (status < 0) {
                mlog(ML_ERROR, "read block %llu failed %d\n",
                     (unsigned long long)blkno, status);
                goto bail;
        }

        inode_fe = (struct ocfs2_dinode *) inode_bh->b_data;
        if (!OCFS2_IS_VALID_DINODE(inode_fe)) {
                mlog(ML_ERROR, "invalid inode %llu requested\n",
                     (unsigned long long)blkno);
                status = -EINVAL;
                goto bail;
        }

        if (le16_to_cpu(inode_fe->i_suballoc_slot) != (u16)OCFS2_INVALID_SLOT &&
            (u32)le16_to_cpu(inode_fe->i_suballoc_slot) > osb->max_slots - 1) {
                mlog(ML_ERROR, "inode %llu has invalid suballoc slot %u\n",
                     (unsigned long long)blkno,
                     (u32)le16_to_cpu(inode_fe->i_suballoc_slot));
                status = -EINVAL;
                goto bail;
        }

        if (suballoc_slot)
                *suballoc_slot = le16_to_cpu(inode_fe->i_suballoc_slot);
        if (suballoc_bit)
                *suballoc_bit = le16_to_cpu(inode_fe->i_suballoc_bit);
        if (group_blkno)
                *group_blkno = le64_to_cpu(inode_fe->i_suballoc_loc);

bail:
        brelse(inode_bh);

        if (status)
                mlog_errno(status);
        return status;
}

/*
 * test whether bit is SET in allocator bitmap or not.  on success, 0
 * is returned and *res is 1 for SET; 0 otherwise.  when fails, errno
 * is returned and *res is meaningless.  Call this after you have
 * cluster locked against suballoc, or you may get a result based on
 * non-up2date contents
 */
static int ocfs2_test_suballoc_bit(struct ocfs2_super *osb,
                                   struct inode *suballoc,
                                   struct buffer_head *alloc_bh,
                                   u64 group_blkno, u64 blkno,
                                   u16 bit, int *res)
{
        struct ocfs2_dinode *alloc_di;
        struct ocfs2_group_desc *group;
        struct buffer_head *group_bh = NULL;
        u64 bg_blkno;
        int status, quiet = 0, released = 0;

        trace_ocfs2_test_suballoc_bit((unsigned long long)blkno,
                                      (unsigned int)bit);

        alloc_di = (struct ocfs2_dinode *)alloc_bh->b_data;
        if ((bit + 1) > ocfs2_bits_per_group(&alloc_di->id2.i_chain)) {
                mlog(ML_ERROR, "suballoc bit %u out of range of %u\n",
                     (unsigned int)bit,
                     ocfs2_bits_per_group(&alloc_di->id2.i_chain));
                status = -EINVAL;
                goto bail;
        }

        bg_blkno = group_blkno ? group_blkno :
                   ocfs2_which_suballoc_group(blkno, bit);
        status = ocfs2_read_hint_group_descriptor(suballoc, alloc_di, bg_blkno,
                                             &group_bh, &released);
        if (released) {
                quiet = 1;
                status = -ESTALE;
                goto bail;
        } else if (status < 0) {
                mlog(ML_ERROR, "read group %llu failed %d\n",
                     (unsigned long long)bg_blkno, status);
                goto bail;
        }

        group = (struct ocfs2_group_desc *) group_bh->b_data;
        *res = ocfs2_test_bit(bit, (unsigned long *)group->bg_bitmap);

bail:
        brelse(group_bh);

        if (status && !quiet)
                mlog_errno(status);
        return status;
}

/*
 * Test if the bit representing this inode (blkno) is set in the
 * suballocator.
 *
 * On success, 0 is returned and *res is 1 for SET; 0 otherwise.
 *
 * In the event of failure, a negative value is returned and *res is
 * meaningless.
 *
 * Callers must make sure to hold nfs_sync_lock to prevent
 * ocfs2_delete_inode() on another node from accessing the same
 * suballocator concurrently.
 */
int ocfs2_test_inode_bit(struct ocfs2_super *osb, u64 blkno, int *res)
{
        int status, quiet = 0;
        u64 group_blkno = 0;
        u16 suballoc_bit = 0, suballoc_slot = 0;
        struct inode *inode_alloc_inode;
        struct buffer_head *alloc_bh = NULL;

        trace_ocfs2_test_inode_bit((unsigned long long)blkno);

        status = ocfs2_get_suballoc_slot_bit(osb, blkno, &suballoc_slot,
                                             &group_blkno, &suballoc_bit);
        if (status < 0) {
                mlog(ML_ERROR, "get alloc slot and bit failed %d\n", status);
                goto bail;
        }

        if (suballoc_slot == (u16)OCFS2_INVALID_SLOT)
                inode_alloc_inode = ocfs2_get_system_file_inode(osb,
                        GLOBAL_INODE_ALLOC_SYSTEM_INODE, suballoc_slot);
        else
                inode_alloc_inode = ocfs2_get_system_file_inode(osb,
                        INODE_ALLOC_SYSTEM_INODE, suballoc_slot);
        if (!inode_alloc_inode) {
                /* the error code could be inaccurate, but we are not able to
                 * get the correct one. */
                status = -EINVAL;
                mlog(ML_ERROR, "unable to get alloc inode in slot %u\n",
                     (u32)suballoc_slot);
                goto bail;
        }

        inode_lock(inode_alloc_inode);
        status = ocfs2_inode_lock(inode_alloc_inode, &alloc_bh, 0);
        if (status < 0) {
                inode_unlock(inode_alloc_inode);
                iput(inode_alloc_inode);
                mlog(ML_ERROR, "lock on alloc inode on slot %u failed %d\n",
                     (u32)suballoc_slot, status);
                goto bail;
        }

        status = ocfs2_test_suballoc_bit(osb, inode_alloc_inode, alloc_bh,
                                         group_blkno, blkno, suballoc_bit, res);
        if (status < 0) {
                if (status == -ESTALE)
                        quiet = 1;
                else
                        mlog(ML_ERROR, "test suballoc bit failed %d\n", status);
        }

        ocfs2_inode_unlock(inode_alloc_inode, 0);
        inode_unlock(inode_alloc_inode);

        iput(inode_alloc_inode);
        brelse(alloc_bh);
bail:
        if (status && !quiet)
                mlog_errno(status);
        return status;
}






















   20 




   20 



   19 
   15 


   19 



   19 




   18 


   19 

   20 








    3 



    3 

    1 
    3 






    3 
















   11 

















   11 
   11 




   10 










    9 









   10 


    3 






   10 
   10 


   10 



   10 

   10 





   10 
    8 








   10 
   10 












   10 
   10 






   10 
    4 







    9 
    3 



    6 
    1 


    5 
    3 


    2 



   10 
   10 


   10 

    7 
    6 












   11 





   11 

   11 
   11 




    2 


    2 


    2 
    2 
    1 
    2 



















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
// SPDX-License-Identifier: GPL-2.0
/*
 *  linux/fs/isofs/dir.c
 *
 *  (C) 1992, 1993, 1994  Eric Youngdale Modified for ISO 9660 filesystem.
 *
 *  (C) 1991  Linus Torvalds - minix filesystem
 *
 *  Steve Beynon                       : Missing last directory entries fixed
 *  (stephen@askone.demon.co.uk)      : 21st June 1996
 *
 *  isofs directory handling functions
 */
#include <linux/gfp.h>
#include <linux/filelock.h>
#include <linux/slab.h>
#include "isofs.h"
#include <linux/fileattr.h>

int isofs_name_translate(struct iso_directory_record *de, char *new, struct inode *inode)
{
        char * old = de->name;
        int len = de->name_len[0];
        int i;

        for (i = 0; i < len; i++) {
                unsigned char c = old[i];
                if (!c)
                        break;

                if (c >= 'A' && c <= 'Z')
                        c |= 0x20;        /* lower case */

                /* Drop trailing '.;1' (ISO 9660:1988 7.5.1 requires period) */
                if (c == '.' && i == len - 3 && old[i + 1] == ';' && old[i + 2] == '1')
                        break;

                /* Drop trailing ';1' */
                if (c == ';' && i == len - 2 && old[i + 1] == '1')
                        break;

                /* Convert remaining ';' to '.' */
                /* Also '/' to '.' (broken Acorn-generated ISO9660 images) */
                if (c == ';' || c == '/')
                        c = '.';

                new[i] = c;
        }
        return i;
}

/* Acorn extensions written by Matthew Wilcox <willy@infradead.org> 1998 */
int get_acorn_filename(struct iso_directory_record *de,
                            char *retname, struct inode *inode)
{
        int std;
        unsigned char *chr;
        int retnamlen = isofs_name_translate(de, retname, inode);

        if (retnamlen == 0)
                return 0;
        std = sizeof(struct iso_directory_record) + de->name_len[0];
        if (std & 1)
                std++;
        if (de->length[0] - std != 32)
                return retnamlen;
        chr = ((unsigned char *) de) + std;
        if (strncmp(chr, "ARCHIMEDES", 10))
                return retnamlen;
        if ((*retname == '_') && ((chr[19] & 1) == 1))
                *retname = '!';
        if (((de->flags[0] & 2) == 0) && (chr[13] == 0xff)
                && ((chr[12] & 0xf0) == 0xf0)) {
                retname[retnamlen] = ',';
                sprintf(retname+retnamlen+1, "%3.3x",
                        ((chr[12] & 0xf) << 8) | chr[11]);
                retnamlen += 4;
        }
        return retnamlen;
}

/*
 * This should _really_ be cleaned up some day..
 */
static int do_isofs_readdir(struct inode *inode, struct file *file,
                struct dir_context *ctx,
                char *tmpname)
{
        unsigned long bufsize = ISOFS_BUFFER_SIZE(inode);
        unsigned char bufbits = ISOFS_BUFFER_BITS(inode);
        unsigned long block, offset, block_saved, offset_saved;
        unsigned long inode_number = 0;        /* Quiet GCC */
        struct buffer_head *bh = NULL;
        int len;
        int map;
        int first_de = 1;
        char *p = NULL;                /* Quiet GCC */
        struct iso_directory_record *de;
        struct isofs_sb_info *sbi = ISOFS_SB(inode->i_sb);

        offset = ctx->pos & (bufsize - 1);
        block = ctx->pos >> bufbits;

        while (ctx->pos < inode->i_size) {
                int de_len;

                if (!bh) {
                        bh = isofs_bread(inode, block);
                        if (!bh)
                                return 0;
                }

                de = (struct iso_directory_record *) (bh->b_data + offset);

                de_len = *(unsigned char *)de;

                /*
                 * If the length byte is zero, we should move on to the next
                 * CDROM sector.  If we are at the end of the directory, we
                 * kick out of the while loop.
                 */

                if (de_len == 0) {
                        brelse(bh);
                        bh = NULL;
                        ctx->pos = (ctx->pos + ISOFS_BLOCK_SIZE) & ~(ISOFS_BLOCK_SIZE - 1);
                        block = ctx->pos >> bufbits;
                        offset = 0;
                        continue;
                }

                block_saved = block;
                offset_saved = offset;
                offset += de_len;

                if (!isofs_dir_record_valid(de, offset_saved, bufsize)) {
                        printk(KERN_NOTICE "iso9660: Corrupted directory entry"
                               " in block %lu of inode %llu\n", block,
                               inode->i_ino);
                        brelse(bh);
                        return -EIO;
                }

                if (first_de) {
                        isofs_normalize_block_and_offset(de,
                                                        &block_saved,
                                                        &offset_saved);
                        inode_number = isofs_get_ino(block_saved,
                                                        offset_saved, bufbits);
                }

                if (de->flags[-sbi->s_high_sierra] & 0x80) {
                        first_de = 0;
                        ctx->pos += de_len;
                        continue;
                }
                first_de = 1;

                /* Handle the case of the '.' directory */
                if (de->name_len[0] == 1 && de->name[0] == 0) {
                        if (!dir_emit_dot(file, ctx))
                                break;
                        ctx->pos += de_len;
                        continue;
                }

                len = 0;

                /* Handle the case of the '..' directory */
                if (de->name_len[0] == 1 && de->name[0] == 1) {
                        if (!dir_emit_dotdot(file, ctx))
                                break;
                        ctx->pos += de_len;
                        continue;
                }

                /* Handle everything else.  Do name translation if there
                   is no Rock Ridge NM field. */

                /*
                 * Do not report hidden files if so instructed, or associated
                 * files unless instructed to do so
                 */
                if ((sbi->s_hide && (de->flags[-sbi->s_high_sierra] & 1)) ||
                    (!sbi->s_showassoc &&
                                (de->flags[-sbi->s_high_sierra] & 4))) {
                        ctx->pos += de_len;
                        continue;
                }

                map = 1;
                if (sbi->s_rock) {
                        len = get_rock_ridge_filename(de, tmpname, inode);
                        if (len != 0) {                /* may be -1 */
                                p = tmpname;
                                map = 0;
                        }
                }
                if (map) {
#ifdef CONFIG_JOLIET
                        if (sbi->s_joliet_level) {
                                len = get_joliet_filename(de, tmpname, inode);
                                p = tmpname;
                        } else
#endif
                        if (sbi->s_mapping == 'a') {
                                len = get_acorn_filename(de, tmpname, inode);
                                p = tmpname;
                        } else
                        if (sbi->s_mapping == 'n') {
                                len = isofs_name_translate(de, tmpname, inode);
                                p = tmpname;
                        } else {
                                p = de->name;
                                len = de->name_len[0];
                        }
                }
                if (len > 0) {
                        if (!dir_emit(ctx, p, len, inode_number, DT_UNKNOWN))
                                break;
                }
                ctx->pos += de_len;
        }
        if (bh)
                brelse(bh);
        return 0;
}

/*
 * Handle allocation of temporary space for name translation and
 * handling split directory entries.. The real work is done by
 * "do_isofs_readdir()".
 */
static int isofs_readdir(struct file *file, struct dir_context *ctx)
{
        int result;
        char *tmpname;
        struct inode *inode = file_inode(file);

        tmpname = kmalloc(1024, GFP_KERNEL);
        if (tmpname == NULL)
                return -ENOMEM;

        result = do_isofs_readdir(inode, file, ctx, tmpname);

        kfree(tmpname);
        return result;
}

int isofs_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
{
        struct isofs_sb_info *sbi = ISOFS_SB(dentry->d_sb);

        if (sbi->s_check == 'r') {
                fa->fsx_xflags |= FS_XFLAG_CASEFOLD;
                fa->flags |= FS_CASEFOLD_FL;
        }
        if (!sbi->s_joliet_level && !sbi->s_rock &&
            (sbi->s_mapping == 'n' || sbi->s_mapping == 'a'))
                fa->fsx_xflags |= FS_XFLAG_CASENONPRESERVING;
        return 0;
}

const struct file_operations isofs_dir_operations =
{
        .llseek = generic_file_llseek,
        .read = generic_read_dir,
        .iterate_shared = isofs_readdir,
        .setlease = generic_setlease,
};

/*
 * directories can handle most operations...
 */
const struct inode_operations isofs_dir_inode_operations =
{
        .lookup = isofs_lookup,
        .fileattr_get = isofs_fileattr_get,
};













































































   27 









































    5 

















    5 












    5 












    5 


    5 



    5 
    5 
    5 
























   31 

   22 

   31 














    4 
   20 















    2 
    2 

    6 


    6 
    6 









    2 





   18 
   20 







   18 

   18 
   20 





















    6 



    5 

    4 

    1 


    5 




    5 





    6 





    1 


























   28 


    4 

   11 
   10 
    9 
    9 





   28 

    6 













    2 
    2 





    2 
    2 
    2 
    2 














   31 




   31 



   23 


   23 


   22 



   16 

   15 


   22 


    1 
   21 


    9 


   15 






   10 



   12 


   12 


   10 
    3 

   10 
    8 

    9 


   13 


   31 










    2 



















    2 
    2 



































































    2 
























































































































































    2 















    1 


    1 






   19 


   19 















  169 
   46 




   37 
   33 

   16 
   22 




  169 










   13 


   13 
   13 


    2 


    2 
    2 

    2 
    2 



   13 














   22 






   22 










   22 



    8 





    8 
    8 

   14 
   14 






   22 

   22 

   22 
    2 





   20 



   19 



    2 
    2 


    2 
    1 





   19 




    1 

   19 



    3 

    3 
   22 












   22 


   22 


























    3 







    2 

    1 

    1 

    1 















    1 




    2 

    3 




    1 





    1 





    1 















































   50 
   43 

   50 



   38 
   34 


    7 

    6 

    7 

    2 

    6 
    5 
    4 

    1 



    7 





























   33 
   33 






   33 






    2 

   33 








   23 




   23 



   20 
   20 


   20 





   20 
   20 









   10 









    1 

   29 





    5 





    3 





    3 






    3 







    3 

    3 


    3 


    1 
    3 





    3 











    3 






    3 


    3 



    3 


    3 


    3 
    1 
    2 


    2 





    3 

    2 

    3 



    3 
    3 



    3 
    3 







    3 


   30 
   24 

   30 
   33 












   34 






    8 


   31 



    1 
























   30 
   29 
   30 

   28 






    3 


    2 


    1 



   28 



   28 

   27 


    2 




    1 


   26 




   25 
   26 
    1 







   28 





   28 





   28 
























   37 


   36 




   29 










   34 






   37 


   37 




   64 
















   63 
   64 
   63 

   64 



    9 

   64 



    4 


   10 






    3 



    9 
    2 











    3 

    4 




   64 




























    4 


    4 
    4 

    4 












    3 
    1 


    2 







   12 













    8 












































   15 









    2 
    2 



    2 






    2 






   11 


   10 


    9 


    8 


    4 
    2 





    4 
    1 

    3 

    8 

    6 




    2 




    2 


    2 

    2 


    2 




    1 









   15 

















   17 

    8 


   16 


   14 
   13 


   13 


   13 
   17 







  222 






































   10 

    6 
    4 


    6 







    6 










































































   69 
































   62 


   59 


   62 



   62 






   62 
   62 


   62 
   62 















   62 
   62 





    3 




























   26 


    2 
    1 





   26 




   24 




   23 
    2 

   23 
    9 

   23 
   11 

   23 
    1 

   23 
    1 

   23 
    1 

   23 
    3 

   23 
    2 

   23 
    4 

   23 
   24 






   20 







   20 
   20 
    3 








   19 


    4 


   15 




   15 
   20 
    3 








    2 
    7 







    7 
    5 



    2 

    1 





    4 
    3 
    3 




    7 



    1 




    1 



    1 

    1 




    1 





    3 



    2 
    3 































  141 








  150 
  150 
  150 
  150 
  150 
  150 
  150 
  150 
  150 


  150 
  150 
  150 
  150 


  150 
    6 












  218 



























































































   44 







   44 

   44 




   44 
















   44 




























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        IPv6 tunneling device
 *        Linux INET6 implementation
 *
 *        Authors:
 *        Ville Nuorvala                <vnuorval@tcs.hut.fi>
 *        Yasuyuki Kozakai        <kozakai@linux-ipv6.org>
 *
 *      Based on:
 *      linux/net/ipv6/sit.c and linux/net/ipv4/ipip.c
 *
 *      RFC 2473
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/capability.h>
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/sockios.h>
#include <linux/icmp.h>
#include <linux/if.h>
#include <linux/in.h>
#include <linux/ip.h>
#include <linux/net.h>
#include <linux/in6.h>
#include <linux/netdevice.h>
#include <linux/if_arp.h>
#include <linux/icmpv6.h>
#include <linux/init.h>
#include <linux/route.h>
#include <linux/rtnetlink.h>
#include <linux/netfilter_ipv6.h>
#include <linux/slab.h>
#include <linux/hash.h>
#include <linux/etherdevice.h>

#include <linux/uaccess.h>
#include <linux/atomic.h>

#include <net/icmp.h>
#include <net/ip.h>
#include <net/ip_tunnels.h>
#include <net/ipv6.h>
#include <net/ip6_route.h>
#include <net/addrconf.h>
#include <net/ip6_tunnel.h>
#include <net/xfrm.h>
#include <net/dsfield.h>
#include <net/inet_ecn.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <net/netdev_lock.h>
#include <net/dst_metadata.h>
#include <net/inet_dscp.h>

MODULE_AUTHOR("Ville Nuorvala");
MODULE_DESCRIPTION("IPv6 tunneling device");
MODULE_LICENSE("GPL");
MODULE_ALIAS_RTNL_LINK("ip6tnl");
MODULE_ALIAS_NETDEV("ip6tnl0");

#define IP6_TUNNEL_MAX_DEST_TLVS    8

#define IP6_TUNNEL_HASH_SIZE_SHIFT  5
#define IP6_TUNNEL_HASH_SIZE (1 << IP6_TUNNEL_HASH_SIZE_SHIFT)

static bool log_ecn_error = true;
module_param(log_ecn_error, bool, 0644);
MODULE_PARM_DESC(log_ecn_error, "Log packets received with corrupted ECN");

static u32 HASH(const struct in6_addr *addr1, const struct in6_addr *addr2)
{
        u32 hash = ipv6_addr_hash(addr1) ^ ipv6_addr_hash(addr2);

        return hash_32(hash, IP6_TUNNEL_HASH_SIZE_SHIFT);
}

static int ip6_tnl_dev_init(struct net_device *dev);
static void ip6_tnl_dev_setup(struct net_device *dev);
static struct rtnl_link_ops ip6_link_ops __read_mostly;

static unsigned int ip6_tnl_net_id __read_mostly;
struct ip6_tnl_net {
        /* the IPv6 tunnel fallback device */
        struct net_device *fb_tnl_dev;
        /* lists for storing tunnels in use */
        struct ip6_tnl __rcu *tnls_r_l[IP6_TUNNEL_HASH_SIZE];
        struct ip6_tnl __rcu *tnls_wc[1];
        struct ip6_tnl __rcu **tnls[2];
        struct ip6_tnl __rcu *collect_md_tun;
};

static inline int ip6_tnl_mpls_supported(void)
{
        return IS_ENABLED(CONFIG_MPLS);
}

/**
 * ip6_tnl_lookup - fetch tunnel matching the end-point addresses
 *   @net: network namespace
 *   @link: ifindex of underlying interface
 *   @remote: the address of the tunnel exit-point
 *   @local: the address of the tunnel entry-point
 *
 * Return:
 *   tunnel matching given end-points if found,
 *   else fallback tunnel if its device is up,
 *   else %NULL
 **/

static struct ip6_tnl *
ip6_tnl_lookup(struct net *net, int link,
               const struct in6_addr *remote, const struct in6_addr *local)
{
        unsigned int hash = HASH(remote, local);
        struct ip6_tnl *t, *cand = NULL;
        struct ip6_tnl_net *ip6n = net_generic(net, ip6_tnl_net_id);
        struct in6_addr any;

        for_each_ip_tunnel_rcu(t, ip6n->tnls_r_l[hash]) {
                if (!ipv6_addr_equal(local, &t->parms.laddr) ||
                    !ipv6_addr_equal(remote, &t->parms.raddr) ||
                    !(t->dev->flags & IFF_UP))
                        continue;

                if (link == t->parms.link)
                        return t;
                else
                        cand = t;
        }

        memset(&any, 0, sizeof(any));
        hash = HASH(&any, local);
        for_each_ip_tunnel_rcu(t, ip6n->tnls_r_l[hash]) {
                if (!ipv6_addr_equal(local, &t->parms.laddr) ||
                    !ipv6_addr_any(&t->parms.raddr) ||
                    !(t->dev->flags & IFF_UP))
                        continue;

                if (link == t->parms.link)
                        return t;
                else if (!cand)
                        cand = t;
        }

        hash = HASH(remote, &any);
        for_each_ip_tunnel_rcu(t, ip6n->tnls_r_l[hash]) {
                if (!ipv6_addr_equal(remote, &t->parms.raddr) ||
                    !ipv6_addr_any(&t->parms.laddr) ||
                    !(t->dev->flags & IFF_UP))
                        continue;

                if (link == t->parms.link)
                        return t;
                else if (!cand)
                        cand = t;
        }

        if (cand)
                return cand;

        t = rcu_dereference(ip6n->collect_md_tun);
        if (t && t->dev->flags & IFF_UP)
                return t;

        t = rcu_dereference(ip6n->tnls_wc[0]);
        if (t && (t->dev->flags & IFF_UP))
                return t;

        return NULL;
}

/**
 * ip6_tnl_bucket - get head of list matching given tunnel parameters
 *   @ip6n: the private data for ip6_vti in the netns
 *   @p: parameters containing tunnel end-points
 *
 * Description:
 *   ip6_tnl_bucket() returns the head of the list matching the
 *   &struct in6_addr entries laddr and raddr in @p.
 *
 * Return: head of IPv6 tunnel list
 **/

static struct ip6_tnl __rcu **
ip6_tnl_bucket(struct ip6_tnl_net *ip6n, const struct __ip6_tnl_parm *p)
{
        const struct in6_addr *remote = &p->raddr;
        const struct in6_addr *local = &p->laddr;
        unsigned int h = 0;
        int prio = 0;

        if (!ipv6_addr_any(remote) || !ipv6_addr_any(local)) {
                prio = 1;
                h = HASH(remote, local);
        }
        return &ip6n->tnls[prio][h];
}

/**
 * ip6_tnl_link - add tunnel to hash table
 *   @ip6n: the private data for ip6_vti in the netns
 *   @t: tunnel to be added
 **/

static void
ip6_tnl_link(struct ip6_tnl_net *ip6n, struct ip6_tnl *t)
{
        struct ip6_tnl __rcu **tp = ip6_tnl_bucket(ip6n, &t->parms);

        if (t->parms.collect_md)
                rcu_assign_pointer(ip6n->collect_md_tun, t);
        rcu_assign_pointer(t->next , rtnl_dereference(*tp));
        rcu_assign_pointer(*tp, t);
}

/**
 * ip6_tnl_unlink - remove tunnel from hash table
 *   @ip6n: the private data for ip6_vti in the netns
 *   @t: tunnel to be removed
 **/

static void
ip6_tnl_unlink(struct ip6_tnl_net *ip6n, struct ip6_tnl *t)
{
        struct ip6_tnl __rcu **tp;
        struct ip6_tnl *iter;

        if (t->parms.collect_md)
                rcu_assign_pointer(ip6n->collect_md_tun, NULL);

        for (tp = ip6_tnl_bucket(ip6n, &t->parms);
             (iter = rtnl_dereference(*tp)) != NULL;
             tp = &iter->next) {
                if (t == iter) {
                        rcu_assign_pointer(*tp, t->next);
                        break;
                }
        }
}

static void ip6_dev_free(struct net_device *dev)
{
        struct ip6_tnl *t = netdev_priv(dev);

        gro_cells_destroy(&t->gro_cells);
        dst_cache_destroy(&t->dst_cache);
}

static int ip6_tnl_create2(struct net_device *dev)
{
        struct ip6_tnl *t = netdev_priv(dev);
        struct ip6_tnl_net *ip6n = net_generic(t->net, ip6_tnl_net_id);
        int err;

        dev->rtnl_link_ops = &ip6_link_ops;
        err = register_netdevice(dev);
        if (err < 0)
                goto out;

        strcpy(t->parms.name, dev->name);

        ip6_tnl_link(ip6n, t);
        return 0;

out:
        return err;
}

/**
 * ip6_tnl_create - create a new tunnel
 *   @net: network namespace
 *   @p: tunnel parameters
 *
 * Description:
 *   Create tunnel matching given parameters.
 *
 * Return:
 *   created tunnel or error pointer
 **/

static struct ip6_tnl *ip6_tnl_create(struct net *net, struct __ip6_tnl_parm *p)
{
        struct net_device *dev;
        struct ip6_tnl *t;
        char name[IFNAMSIZ];
        int err = -E2BIG;

        if (p->name[0]) {
                if (!dev_valid_name(p->name))
                        goto failed;
                strscpy(name, p->name, IFNAMSIZ);
        } else {
                sprintf(name, "ip6tnl%%d");
        }
        err = -ENOMEM;
        dev = alloc_netdev(sizeof(*t), name, NET_NAME_UNKNOWN,
                           ip6_tnl_dev_setup);
        if (!dev)
                goto failed;

        dev_net_set(dev, net);

        t = netdev_priv(dev);
        t->parms = *p;
        t->net = dev_net(dev);
        err = ip6_tnl_create2(dev);
        if (err < 0)
                goto failed_free;

        return t;

failed_free:
        free_netdev(dev);
failed:
        return ERR_PTR(err);
}

/**
 * ip6_tnl_locate - find or create tunnel matching given parameters
 *   @net: network namespace
 *   @p: tunnel parameters
 *   @create: != 0 if allowed to create new tunnel if no match found
 *
 * Description:
 *   ip6_tnl_locate() first tries to locate an existing tunnel
 *   based on @parms. If this is unsuccessful, but @create is set a new
 *   tunnel device is created and registered for use.
 *
 * Return:
 *   matching tunnel or error pointer
 **/

static struct ip6_tnl *ip6_tnl_locate(struct net *net,
                struct __ip6_tnl_parm *p, int create)
{
        const struct in6_addr *remote = &p->raddr;
        const struct in6_addr *local = &p->laddr;
        struct ip6_tnl __rcu **tp;
        struct ip6_tnl *t;
        struct ip6_tnl_net *ip6n = net_generic(net, ip6_tnl_net_id);

        for (tp = ip6_tnl_bucket(ip6n, p);
             (t = rtnl_dereference(*tp)) != NULL;
             tp = &t->next) {
                if (ipv6_addr_equal(local, &t->parms.laddr) &&
                    ipv6_addr_equal(remote, &t->parms.raddr) &&
                    p->link == t->parms.link) {
                        if (create)
                                return ERR_PTR(-EEXIST);

                        return t;
                }
        }
        if (!create)
                return ERR_PTR(-ENODEV);
        return ip6_tnl_create(net, p);
}

/**
 * ip6_tnl_dev_uninit - tunnel device uninitializer
 *   @dev: the device to be destroyed
 *
 * Description:
 *   ip6_tnl_dev_uninit() removes tunnel from its list
 **/

static void
ip6_tnl_dev_uninit(struct net_device *dev)
{
        struct ip6_tnl *t = netdev_priv(dev);
        struct net *net = t->net;
        struct ip6_tnl_net *ip6n = net_generic(net, ip6_tnl_net_id);

        if (dev == ip6n->fb_tnl_dev)
                RCU_INIT_POINTER(ip6n->tnls_wc[0], NULL);
        else
                ip6_tnl_unlink(ip6n, t);
        dst_cache_reset(&t->dst_cache);
        netdev_put(dev, &t->dev_tracker);
}

/**
 * ip6_tnl_parse_tlv_enc_lim - handle encapsulation limit option
 *   @skb: received socket buffer
 *   @raw: the ICMPv6 error message data
 *
 * Return:
 *   0 if none was found,
 *   else index to encapsulation limit
 **/

__u16 ip6_tnl_parse_tlv_enc_lim(struct sk_buff *skb, __u8 *raw)
{
        const struct ipv6hdr *ipv6h = (const struct ipv6hdr *)raw;
        unsigned int nhoff = raw - skb->data;
        unsigned int off = nhoff + sizeof(*ipv6h);
        u8 nexthdr = ipv6h->nexthdr;
        int exthdr_cnt = 0;

        while (ipv6_ext_hdr(nexthdr) && nexthdr != NEXTHDR_NONE) {
                struct ipv6_opt_hdr *hdr;
                u16 optlen;

                if (unlikely(exthdr_cnt++ >= IP6_MAX_EXT_HDRS_CNT))
                        break;

                if (!pskb_may_pull(skb, off + sizeof(*hdr)))
                        break;

                hdr = (struct ipv6_opt_hdr *)(skb->data + off);
                if (nexthdr == NEXTHDR_FRAGMENT) {
                        optlen = 8;
                } else if (nexthdr == NEXTHDR_AUTH) {
                        optlen = ipv6_authlen(hdr);
                } else {
                        optlen = ipv6_optlen(hdr);
                }

                if (!pskb_may_pull(skb, off + optlen))
                        break;

                hdr = (struct ipv6_opt_hdr *)(skb->data + off);
                if (nexthdr == NEXTHDR_FRAGMENT) {
                        struct frag_hdr *frag_hdr = (struct frag_hdr *)hdr;

                        if (frag_hdr->frag_off)
                                break;
                }
                if (nexthdr == NEXTHDR_DEST) {
                        int tlv_cnt = 0;
                        u16 i = 2;

                        while (1) {
                                struct ipv6_tlv_tnl_enc_lim *tel;

                                if (unlikely(tlv_cnt++ >= IP6_TUNNEL_MAX_DEST_TLVS))
                                        break;

                                /* No more room for encapsulation limit */
                                if (i + sizeof(*tel) > optlen)
                                        break;

                                tel = (struct ipv6_tlv_tnl_enc_lim *)(skb->data + off + i);
                                /* return index of option if found and valid */
                                if (tel->type == IPV6_TLV_TNL_ENCAP_LIMIT &&
                                    tel->length == 1)
                                        return i + off - nhoff;
                                /* else jump to next option */
                                if (tel->type)
                                        i += tel->length + 2;
                                else
                                        i++;
                        }
                }
                nexthdr = hdr->nexthdr;
                off += optlen;
        }
        return 0;
}
EXPORT_SYMBOL(ip6_tnl_parse_tlv_enc_lim);

/* ip6_tnl_err() should handle errors in the tunnel according to the
 * specifications in RFC 2473.
 */
static int
ip6_tnl_err(struct sk_buff *skb, __u8 ipproto, struct inet6_skb_parm *opt,
            u8 *type, u8 *code, int *msg, __u32 *info, int offset)
{
        const struct ipv6hdr *ipv6h = (const struct ipv6hdr *)skb->data;
        struct net *net = dev_net(skb->dev);
        u8 rel_type = ICMPV6_DEST_UNREACH;
        u8 rel_code = ICMPV6_ADDR_UNREACH;
        __u32 rel_info = 0;
        struct ip6_tnl *t;
        int err = -ENOENT;
        int rel_msg = 0;
        u8 tproto;
        __u16 len;

        /* If the packet doesn't contain the original IPv6 header we are
           in trouble since we might need the source address for further
           processing of the error. */

        rcu_read_lock();
        t = ip6_tnl_lookup(dev_net(skb->dev), skb->dev->ifindex, &ipv6h->daddr, &ipv6h->saddr);
        if (!t)
                goto out;

        tproto = READ_ONCE(t->parms.proto);
        if (tproto != ipproto && tproto != 0)
                goto out;

        err = 0;

        switch (*type) {
        case ICMPV6_DEST_UNREACH:
                net_dbg_ratelimited("%s: Path to destination invalid or inactive!\n",
                                    t->parms.name);
                rel_msg = 1;
                break;
        case ICMPV6_TIME_EXCEED:
                if ((*code) == ICMPV6_EXC_HOPLIMIT) {
                        net_dbg_ratelimited("%s: Too small hop limit or routing loop in tunnel!\n",
                                            t->parms.name);
                        rel_msg = 1;
                }
                break;
        case ICMPV6_PARAMPROB: {
                struct ipv6_tlv_tnl_enc_lim *tel;
                __u32 teli;

                teli = 0;
                if ((*code) == ICMPV6_HDR_FIELD)
                        teli = ip6_tnl_parse_tlv_enc_lim(skb, skb->data);

                if (teli && teli == *info - 2) {
                        tel = (struct ipv6_tlv_tnl_enc_lim *) &skb->data[teli];
                        if (tel->encap_limit == 0) {
                                net_dbg_ratelimited("%s: Too small encapsulation limit or routing loop in tunnel!\n",
                                                    t->parms.name);
                                rel_msg = 1;
                        }
                } else {
                        net_dbg_ratelimited("%s: Recipient unable to parse tunneled packet!\n",
                                            t->parms.name);
                }
                break;
        }
        case ICMPV6_PKT_TOOBIG: {
                __u32 mtu;

                ip6_update_pmtu(skb, net, htonl(*info), 0, 0,
                                sock_net_uid(net, NULL));
                mtu = *info - offset;
                if (mtu < IPV6_MIN_MTU)
                        mtu = IPV6_MIN_MTU;
                len = sizeof(*ipv6h) + ntohs(ipv6h->payload_len);
                if (len > mtu) {
                        rel_type = ICMPV6_PKT_TOOBIG;
                        rel_code = 0;
                        rel_info = mtu;
                        rel_msg = 1;
                }
                break;
        }
        case NDISC_REDIRECT:
                ip6_redirect(skb, net, skb->dev->ifindex, 0,
                             sock_net_uid(net, NULL));
                break;
        }

        *type = rel_type;
        *code = rel_code;
        *info = rel_info;
        *msg = rel_msg;

out:
        rcu_read_unlock();
        return err;
}

static int
ip4ip6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
           u8 type, u8 code, int offset, __be32 info)
{
        __u32 rel_info = ntohl(info);
        const struct iphdr *eiph;
        struct sk_buff *skb2;
        int err, rel_msg = 0;
        u8 rel_type = type;
        u8 rel_code = code;
        struct rtable *rt;
        struct flowi4 fl4;

        err = ip6_tnl_err(skb, IPPROTO_IPIP, opt, &rel_type, &rel_code,
                          &rel_msg, &rel_info, offset);
        if (err < 0)
                return err;

        if (rel_msg == 0)
                return 0;

        switch (rel_type) {
        case ICMPV6_DEST_UNREACH:
                if (rel_code != ICMPV6_ADDR_UNREACH)
                        return 0;
                rel_type = ICMP_DEST_UNREACH;
                rel_code = ICMP_HOST_UNREACH;
                break;
        case ICMPV6_PKT_TOOBIG:
                if (rel_code != 0)
                        return 0;
                rel_type = ICMP_DEST_UNREACH;
                rel_code = ICMP_FRAG_NEEDED;
                break;
        default:
                return 0;
        }

        if (!pskb_may_pull(skb, offset + sizeof(struct iphdr)))
                return 0;

        skb2 = skb_clone(skb, GFP_ATOMIC);
        if (!skb2)
                return 0;

        /* Remove debris left by IPv6 stack. */
        memset(IPCB(skb2), 0, sizeof(*IPCB(skb2)));

        skb_dst_drop(skb2);

        skb_pull(skb2, offset);
        skb_reset_network_header(skb2);
        eiph = ip_hdr(skb2);
        if (eiph->version != 4 || eiph->ihl < 5)
                goto out;

        /* Try to guess incoming interface */
        rt = ip_route_output_ports(dev_net(skb->dev), &fl4, NULL, eiph->saddr,
                                   0, 0, 0, IPPROTO_IPIP,
                                   eiph->tos & INET_DSCP_MASK, 0);
        if (IS_ERR(rt))
                goto out;

        skb2->dev = rt->dst.dev;
        ip_rt_put(rt);

        /* route "incoming" packet */
        if (rt->rt_flags & RTCF_LOCAL) {
                rt = ip_route_output_ports(dev_net(skb->dev), &fl4, NULL,
                                           eiph->daddr, eiph->saddr, 0, 0,
                                           IPPROTO_IPIP,
                                           eiph->tos & INET_DSCP_MASK, 0);
                if (IS_ERR(rt) || rt->dst.dev->type != ARPHRD_TUNNEL6) {
                        if (!IS_ERR(rt))
                                ip_rt_put(rt);
                        goto out;
                }
                skb_dst_set(skb2, &rt->dst);
        } else {
                if (ip_route_input(skb2, eiph->daddr, eiph->saddr,
                                   ip4h_dscp(eiph), skb2->dev) ||
                    skb_dst_dev(skb2)->type != ARPHRD_TUNNEL6)
                        goto out;
        }

        /* change mtu on this route */
        if (rel_type == ICMP_DEST_UNREACH && rel_code == ICMP_FRAG_NEEDED) {
                if (rel_info > dst4_mtu(skb_dst(skb2)))
                        goto out;

                skb_dst_update_pmtu_no_confirm(skb2, rel_info);
        }

        icmp_send(skb2, rel_type, rel_code, htonl(rel_info));

out:
        kfree_skb(skb2);
        return 0;
}

static int
ip6ip6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
           u8 type, u8 code, int offset, __be32 info)
{
        __u32 rel_info = ntohl(info);
        int err, rel_msg = 0;
        u8 rel_type = type;
        u8 rel_code = code;

        err = ip6_tnl_err(skb, IPPROTO_IPV6, opt, &rel_type, &rel_code,
                          &rel_msg, &rel_info, offset);
        if (err < 0)
                return err;

        if (rel_msg && pskb_may_pull(skb, offset + sizeof(struct ipv6hdr))) {
                struct rt6_info *rt;
                struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);

                if (!skb2)
                        return 0;

                /* Remove debris left by outer IPv6 stack. */
                memset(IP6CB(skb2), 0, sizeof(*IP6CB(skb2)));

                skb_dst_drop(skb2);
                skb_pull(skb2, offset);
                skb_reset_network_header(skb2);

                /* Try to guess incoming interface */
                rt = rt6_lookup(dev_net(skb->dev), &ipv6_hdr(skb2)->saddr,
                                NULL, 0, skb2, 0);

                if (rt && rt->dst.dev)
                        skb2->dev = rt->dst.dev;

                icmpv6_send(skb2, rel_type, rel_code, rel_info);

                ip6_rt_put(rt);

                kfree_skb(skb2);
        }

        return 0;
}

static int
mplsip6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
            u8 type, u8 code, int offset, __be32 info)
{
        __u32 rel_info = ntohl(info);
        int err, rel_msg = 0;
        u8 rel_type = type;
        u8 rel_code = code;

        err = ip6_tnl_err(skb, IPPROTO_MPLS, opt, &rel_type, &rel_code,
                          &rel_msg, &rel_info, offset);
        return err;
}

static int ip4ip6_dscp_ecn_decapsulate(const struct ip6_tnl *t,
                                       const struct ipv6hdr *ipv6h,
                                       struct sk_buff *skb)
{
        __u8 dsfield = ipv6_get_dsfield(ipv6h) & ~INET_ECN_MASK;

        if (t->parms.flags & IP6_TNL_F_RCV_DSCP_COPY)
                ipv4_change_dsfield(ip_hdr(skb), INET_ECN_MASK, dsfield);

        return IP6_ECN_decapsulate(ipv6h, skb);
}

static int ip6ip6_dscp_ecn_decapsulate(const struct ip6_tnl *t,
                                       const struct ipv6hdr *ipv6h,
                                       struct sk_buff *skb)
{
        if (t->parms.flags & IP6_TNL_F_RCV_DSCP_COPY)
                ipv6_copy_dscp(ipv6_get_dsfield(ipv6h), ipv6_hdr(skb));

        return IP6_ECN_decapsulate(ipv6h, skb);
}

static inline int mplsip6_dscp_ecn_decapsulate(const struct ip6_tnl *t,
                                               const struct ipv6hdr *ipv6h,
                                               struct sk_buff *skb)
{
        /* ECN is not supported in AF_MPLS */
        return 0;
}

__u32 ip6_tnl_get_cap(struct ip6_tnl *t,
                             const struct in6_addr *laddr,
                             const struct in6_addr *raddr)
{
        struct __ip6_tnl_parm *p = &t->parms;
        int ltype = ipv6_addr_type(laddr);
        int rtype = ipv6_addr_type(raddr);
        __u32 flags = 0;

        if (ltype == IPV6_ADDR_ANY || rtype == IPV6_ADDR_ANY) {
                flags = IP6_TNL_F_CAP_PER_PACKET;
        } else if (ltype & (IPV6_ADDR_UNICAST|IPV6_ADDR_MULTICAST) &&
                   rtype & (IPV6_ADDR_UNICAST|IPV6_ADDR_MULTICAST) &&
                   !((ltype|rtype) & IPV6_ADDR_LOOPBACK) &&
                   (!((ltype|rtype) & IPV6_ADDR_LINKLOCAL) || p->link)) {
                if (ltype&IPV6_ADDR_UNICAST)
                        flags |= IP6_TNL_F_CAP_XMIT;
                if (rtype&IPV6_ADDR_UNICAST)
                        flags |= IP6_TNL_F_CAP_RCV;
        }
        return flags;
}
EXPORT_SYMBOL(ip6_tnl_get_cap);

/* called with rcu_read_lock() */
int ip6_tnl_rcv_ctl(struct ip6_tnl *t,
                                  const struct in6_addr *laddr,
                                  const struct in6_addr *raddr)
{
        struct __ip6_tnl_parm *p = &t->parms;
        int ret = 0;
        struct net *net = t->net;

        if ((p->flags & IP6_TNL_F_CAP_RCV) ||
            ((p->flags & IP6_TNL_F_CAP_PER_PACKET) &&
             (ip6_tnl_get_cap(t, laddr, raddr) & IP6_TNL_F_CAP_RCV))) {
                struct net_device *ldev = NULL;

                if (p->link)
                        ldev = dev_get_by_index_rcu(net, p->link);

                if ((ipv6_addr_is_multicast(laddr) ||
                     likely(ipv6_chk_addr_and_flags(net, laddr, ldev, false,
                                                    0, IFA_F_TENTATIVE))) &&
                    ((p->flags & IP6_TNL_F_ALLOW_LOCAL_REMOTE) ||
                     likely(!ipv6_chk_addr_and_flags(net, raddr, ldev, true,
                                                     0, IFA_F_TENTATIVE))))
                        ret = 1;
        }
        return ret;
}
EXPORT_SYMBOL_GPL(ip6_tnl_rcv_ctl);

static int __ip6_tnl_rcv(struct ip6_tnl *tunnel, struct sk_buff *skb,
                         const struct tnl_ptk_info *tpi,
                         struct metadata_dst *tun_dst,
                         int (*dscp_ecn_decapsulate)(const struct ip6_tnl *t,
                                                const struct ipv6hdr *ipv6h,
                                                struct sk_buff *skb),
                         bool log_ecn_err)
{
        const struct ipv6hdr *ipv6h;
        int nh, err;

        if (test_bit(IP_TUNNEL_CSUM_BIT, tunnel->parms.i_flags) !=
            test_bit(IP_TUNNEL_CSUM_BIT, tpi->flags)) {
                DEV_STATS_INC(tunnel->dev, rx_crc_errors);
                DEV_STATS_INC(tunnel->dev, rx_errors);
                goto drop;
        }

        if (test_bit(IP_TUNNEL_SEQ_BIT, tunnel->parms.i_flags)) {
                if (!test_bit(IP_TUNNEL_SEQ_BIT, tpi->flags) ||
                    (tunnel->i_seqno &&
                     (s32)(ntohl(tpi->seq) - tunnel->i_seqno) < 0)) {
                        DEV_STATS_INC(tunnel->dev, rx_fifo_errors);
                        DEV_STATS_INC(tunnel->dev, rx_errors);
                        goto drop;
                }
                tunnel->i_seqno = ntohl(tpi->seq) + 1;
        }

        skb->protocol = tpi->proto;

        /* Warning: All skb pointers will be invalidated! */
        if (tunnel->dev->type == ARPHRD_ETHER) {
                if (!pskb_may_pull(skb, ETH_HLEN)) {
                        DEV_STATS_INC(tunnel->dev, rx_length_errors);
                        DEV_STATS_INC(tunnel->dev, rx_errors);
                        goto drop;
                }

                skb->protocol = eth_type_trans(skb, tunnel->dev);
                skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
        } else {
                skb->dev = tunnel->dev;
                skb_reset_mac_header(skb);
        }

        /* Save offset of outer header relative to skb->head,
         * because we are going to reset the network header to the inner header
         * and might change skb->head.
         */
        nh = skb_network_header(skb) - skb->head;

        skb_reset_network_header(skb);

        if (skb_vlan_inet_prepare(skb, true)) {
                DEV_STATS_INC(tunnel->dev, rx_length_errors);
                DEV_STATS_INC(tunnel->dev, rx_errors);
                goto drop;
        }

        /* Get the outer header. */
        ipv6h = (struct ipv6hdr *)(skb->head + nh);

        memset(skb->cb, 0, sizeof(struct inet6_skb_parm));

        __skb_tunnel_rx(skb, tunnel->dev, tunnel->net);

        err = dscp_ecn_decapsulate(tunnel, ipv6h, skb);
        if (unlikely(err)) {
                if (log_ecn_err)
                        net_info_ratelimited("non-ECT from %pI6 with DS=%#x\n",
                                             &ipv6h->saddr,
                                             ipv6_get_dsfield(ipv6h));
                if (err > 1) {
                        DEV_STATS_INC(tunnel->dev, rx_frame_errors);
                        DEV_STATS_INC(tunnel->dev, rx_errors);
                        goto drop;
                }
        }

        dev_sw_netstats_rx_add(tunnel->dev, skb->len);

        skb_scrub_packet(skb, !net_eq(tunnel->net, dev_net(tunnel->dev)));

        if (tun_dst)
                skb_dst_set(skb, (struct dst_entry *)tun_dst);

        gro_cells_receive(&tunnel->gro_cells, skb);
        return 0;

drop:
        if (tun_dst)
                dst_release((struct dst_entry *)tun_dst);
        kfree_skb(skb);
        return 0;
}

int ip6_tnl_rcv(struct ip6_tnl *t, struct sk_buff *skb,
                const struct tnl_ptk_info *tpi,
                struct metadata_dst *tun_dst,
                bool log_ecn_err)
{
        int (*dscp_ecn_decapsulate)(const struct ip6_tnl *t,
                                    const struct ipv6hdr *ipv6h,
                                    struct sk_buff *skb);

        dscp_ecn_decapsulate = ip6ip6_dscp_ecn_decapsulate;
        if (tpi->proto == htons(ETH_P_IP))
                dscp_ecn_decapsulate = ip4ip6_dscp_ecn_decapsulate;

        return __ip6_tnl_rcv(t, skb, tpi, tun_dst, dscp_ecn_decapsulate,
                             log_ecn_err);
}
EXPORT_SYMBOL(ip6_tnl_rcv);

static const struct tnl_ptk_info tpi_v6 = {
        /* no tunnel info required for ipxip6. */
        .proto = htons(ETH_P_IPV6),
};

static const struct tnl_ptk_info tpi_v4 = {
        /* no tunnel info required for ipxip6. */
        .proto = htons(ETH_P_IP),
};

static const struct tnl_ptk_info tpi_mpls = {
        /* no tunnel info required for mplsip6. */
        .proto = htons(ETH_P_MPLS_UC),
};

static int ipxip6_rcv(struct sk_buff *skb, u8 ipproto,
                      const struct tnl_ptk_info *tpi,
                      int (*dscp_ecn_decapsulate)(const struct ip6_tnl *t,
                                                  const struct ipv6hdr *ipv6h,
                                                  struct sk_buff *skb))
{
        struct ip6_tnl *t;
        const struct ipv6hdr *ipv6h = ipv6_hdr(skb);
        struct metadata_dst *tun_dst = NULL;
        int ret = -1;

        rcu_read_lock();
        t = ip6_tnl_lookup(dev_net(skb->dev), skb->dev->ifindex, &ipv6h->saddr, &ipv6h->daddr);

        if (t) {
                u8 tproto = READ_ONCE(t->parms.proto);

                if (tproto != ipproto && tproto != 0)
                        goto drop;
                if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb))
                        goto drop;
                ipv6h = ipv6_hdr(skb);
                if (!ip6_tnl_rcv_ctl(t, &ipv6h->daddr, &ipv6h->saddr))
                        goto drop;
                if (iptunnel_pull_header(skb, 0, tpi->proto, false))
                        goto drop;
                if (t->parms.collect_md) {
                        IP_TUNNEL_DECLARE_FLAGS(flags) = { };

                        tun_dst = ipv6_tun_rx_dst(skb, flags, 0, 0);
                        if (!tun_dst)
                                goto drop;
                }
                ret = __ip6_tnl_rcv(t, skb, tpi, tun_dst, dscp_ecn_decapsulate,
                                    log_ecn_error);
        }

        rcu_read_unlock();

        return ret;

drop:
        rcu_read_unlock();
        kfree_skb(skb);
        return 0;
}

static int ip4ip6_rcv(struct sk_buff *skb)
{
        return ipxip6_rcv(skb, IPPROTO_IPIP, &tpi_v4,
                          ip4ip6_dscp_ecn_decapsulate);
}

static int ip6ip6_rcv(struct sk_buff *skb)
{
        return ipxip6_rcv(skb, IPPROTO_IPV6, &tpi_v6,
                          ip6ip6_dscp_ecn_decapsulate);
}

static int mplsip6_rcv(struct sk_buff *skb)
{
        return ipxip6_rcv(skb, IPPROTO_MPLS, &tpi_mpls,
                          mplsip6_dscp_ecn_decapsulate);
}

struct ipv6_tel_txoption {
        struct ipv6_txoptions ops;
        __u8 dst_opt[8];
};

static void init_tel_txopt(struct ipv6_tel_txoption *opt, __u8 encap_limit)
{
        memset(opt, 0, sizeof(struct ipv6_tel_txoption));

        opt->dst_opt[2] = IPV6_TLV_TNL_ENCAP_LIMIT;
        opt->dst_opt[3] = 1;
        opt->dst_opt[4] = encap_limit;
        opt->dst_opt[5] = IPV6_TLV_PADN;
        opt->dst_opt[6] = 1;

        opt->ops.dst1opt = (struct ipv6_opt_hdr *) opt->dst_opt;
        opt->ops.opt_nflen = 8;
}

/**
 * ip6_tnl_addr_conflict - compare packet addresses to tunnel's own
 *   @t: the outgoing tunnel device
 *   @hdr: IPv6 header from the incoming packet
 *
 * Description:
 *   Avoid trivial tunneling loop by checking that tunnel exit-point
 *   doesn't match source of incoming packet.
 *
 * Return:
 *   1 if conflict,
 *   0 else
 **/

static inline bool
ip6_tnl_addr_conflict(const struct ip6_tnl *t, const struct ipv6hdr *hdr)
{
        return ipv6_addr_equal(&t->parms.raddr, &hdr->saddr);
}

int ip6_tnl_xmit_ctl(struct ip6_tnl *t,
                     const struct in6_addr *laddr,
                     const struct in6_addr *raddr)
{
        struct __ip6_tnl_parm *p = &t->parms;
        int ret = 0;
        struct net *net = t->net;

        if (t->parms.collect_md)
                return 1;

        if ((p->flags & IP6_TNL_F_CAP_XMIT) ||
            ((p->flags & IP6_TNL_F_CAP_PER_PACKET) &&
             (ip6_tnl_get_cap(t, laddr, raddr) & IP6_TNL_F_CAP_XMIT))) {
                struct net_device *ldev = NULL;

                rcu_read_lock();
                if (p->link)
                        ldev = dev_get_by_index_rcu(net, p->link);

                if (unlikely(!ipv6_chk_addr_and_flags(net, laddr, ldev, false,
                                                      0, IFA_F_TENTATIVE)))
                        pr_warn_ratelimited("%s xmit: Local address not yet configured!\n",
                                            p->name);
                else if (!(p->flags & IP6_TNL_F_ALLOW_LOCAL_REMOTE) &&
                         !ipv6_addr_is_multicast(raddr) &&
                         unlikely(ipv6_chk_addr_and_flags(net, raddr, ldev,
                                                          true, 0, IFA_F_TENTATIVE)))
                        pr_warn_ratelimited("%s xmit: Routing loop! Remote address found on this node!\n",
                                            p->name);
                else
                        ret = 1;
                rcu_read_unlock();
        }
        return ret;
}
EXPORT_SYMBOL_GPL(ip6_tnl_xmit_ctl);

/**
 * ip6_tnl_xmit - encapsulate packet and send
 *   @skb: the outgoing socket buffer
 *   @dev: the outgoing tunnel device
 *   @dsfield: dscp code for outer header
 *   @fl6: flow of tunneled packet
 *   @encap_limit: encapsulation limit
 *   @pmtu: Path MTU is stored if packet is too big
 *   @proto: next header value
 *
 * Description:
 *   Build new header and do some sanity checks on the packet before sending
 *   it.
 *
 * Return:
 *   0 on success
 *   -1 fail
 *   %-EMSGSIZE message too big. return mtu in this case.
 **/

int ip6_tnl_xmit(struct sk_buff *skb, struct net_device *dev, __u8 dsfield,
                 struct flowi6 *fl6, int encap_limit, __u32 *pmtu,
                 __u8 proto)
{
        struct ip6_tnl *t = netdev_priv(dev);
        struct net *net = t->net;
        struct ipv6hdr *ipv6h;
        struct ipv6_tel_txoption opt;
        struct dst_entry *dst = NULL, *ndst = NULL;
        struct net_device *tdev;
        int err_count, mtu;
        unsigned int eth_hlen = t->dev->type == ARPHRD_ETHER ? ETH_HLEN : 0;
        unsigned int psh_hlen = sizeof(struct ipv6hdr) + t->encap_hlen;
        unsigned int max_headroom = psh_hlen;
        __be16 payload_protocol;
        bool use_cache = false;
        u8 hop_limit;
        int err = -1;

        payload_protocol = skb_protocol(skb, true);

        if (t->parms.collect_md) {
                hop_limit = skb_tunnel_info(skb)->key.ttl;
                goto route_lookup;
        } else {
                hop_limit = t->parms.hop_limit;
        }

        /* NBMA tunnel */
        if (ipv6_addr_any(&t->parms.raddr)) {
                if (payload_protocol == htons(ETH_P_IPV6)) {
                        struct in6_addr *addr6;
                        struct neighbour *neigh;
                        int addr_type;

                        if (!skb_dst(skb))
                                goto tx_err_link_failure;

                        neigh = dst_neigh_lookup(skb_dst(skb),
                                                 &ipv6_hdr(skb)->daddr);
                        if (!neigh)
                                goto tx_err_link_failure;

                        addr6 = (struct in6_addr *)&neigh->primary_key;
                        addr_type = ipv6_addr_type(addr6);

                        if (addr_type == IPV6_ADDR_ANY)
                                addr6 = &ipv6_hdr(skb)->daddr;

                        memcpy(&fl6->daddr, addr6, sizeof(fl6->daddr));
                        neigh_release(neigh);
                } else if (payload_protocol == htons(ETH_P_IP)) {
                        const struct rtable *rt = skb_rtable(skb);

                        if (!rt)
                                goto tx_err_link_failure;

                        if (rt->rt_gw_family == AF_INET6)
                                memcpy(&fl6->daddr, &rt->rt_gw6, sizeof(fl6->daddr));
                }
        } else if (t->parms.proto != 0 && !(t->parms.flags &
                                            (IP6_TNL_F_USE_ORIG_TCLASS |
                                             IP6_TNL_F_USE_ORIG_FWMARK))) {
                /* enable the cache only if neither the outer protocol nor the
                 * routing decision depends on the current inner header value
                 */
                use_cache = true;
        }

        if (use_cache)
                dst = dst_cache_get(&t->dst_cache);

        if (!ip6_tnl_xmit_ctl(t, &fl6->saddr, &fl6->daddr))
                goto tx_err_link_failure;

        if (!dst) {
route_lookup:
                /* add dsfield to flowlabel for route lookup */
                fl6->flowlabel = ip6_make_flowinfo(dsfield, fl6->flowlabel);

                dst = ip6_route_output(net, NULL, fl6);

                if (dst->error)
                        goto tx_err_link_failure;
                dst = xfrm_lookup(net, dst, flowi6_to_flowi(fl6), NULL, 0);
                if (IS_ERR(dst)) {
                        err = PTR_ERR(dst);
                        dst = NULL;
                        goto tx_err_link_failure;
                }
                if (t->parms.collect_md && ipv6_addr_any(&fl6->saddr) &&
                    ipv6_dev_get_saddr(net, ip6_dst_idev(dst)->dev,
                                       &fl6->daddr, 0, &fl6->saddr))
                        goto tx_err_link_failure;
                ndst = dst;
        }

        tdev = dst_dev(dst);

        if (tdev == dev) {
                DEV_STATS_INC(dev, collisions);
                net_warn_ratelimited("%s: Local routing loop detected!\n",
                                     t->parms.name);
                goto tx_err_dst_release;
        }
        mtu = dst6_mtu(dst) - eth_hlen - psh_hlen - t->tun_hlen;
        if (encap_limit >= 0) {
                max_headroom += 8;
                mtu -= 8;
        }
        mtu = max(mtu, skb->protocol == htons(ETH_P_IPV6) ?
                       IPV6_MIN_MTU : IPV4_MIN_MTU);

        skb_dst_update_pmtu_no_confirm(skb, mtu);
        if (skb->len - t->tun_hlen - eth_hlen > mtu && !skb_is_gso(skb)) {
                *pmtu = mtu;
                err = -EMSGSIZE;
                goto tx_err_dst_release;
        }

        err_count = READ_ONCE(t->err_count);
        if (err_count > 0) {
                if (time_before(jiffies,
                                READ_ONCE(t->err_time) + IP6TUNNEL_ERR_TIMEO)) {
                        WRITE_ONCE(t->err_count, err_count - 1);

                        dst_link_failure(skb);
                } else {
                        WRITE_ONCE(t->err_count, 0);
                }
        }

        skb_scrub_packet(skb, !net_eq(t->net, dev_net(dev)));

        /*
         * Okay, now see if we can stuff it in the buffer as-is.
         */
        max_headroom += LL_RESERVED_SPACE(tdev);

        if (skb_cow_head(skb, max_headroom))
                goto tx_err_dst_release;

        if (t->parms.collect_md) {
                if (t->encap.type != TUNNEL_ENCAP_NONE)
                        goto tx_err_dst_release;
        } else {
                if (use_cache && ndst)
                        dst_cache_set_ip6(&t->dst_cache, ndst, &fl6->saddr);
        }
        skb_dst_set(skb, dst);

        if (hop_limit == 0) {
                if (payload_protocol == htons(ETH_P_IP))
                        hop_limit = ip_hdr(skb)->ttl;
                else if (payload_protocol == htons(ETH_P_IPV6))
                        hop_limit = ipv6_hdr(skb)->hop_limit;
                else
                        hop_limit = ip6_dst_hoplimit(dst);
        }

        /* Calculate max headroom for all the headers and adjust
         * needed_headroom if necessary.
         */
        max_headroom = LL_RESERVED_SPACE(tdev) + sizeof(struct ipv6hdr)
                        + dst->header_len + t->hlen;
        ip_tunnel_adj_headroom(dev, max_headroom);

        err = ip6_tnl_encap(skb, t, &proto, fl6);
        if (err)
                return err;

        if (encap_limit >= 0) {
                init_tel_txopt(&opt, encap_limit);
                proto = ipv6_push_frag_opts(skb, &opt.ops, proto);
        }

        skb_push(skb, sizeof(struct ipv6hdr));
        skb_reset_network_header(skb);
        ipv6h = ipv6_hdr(skb);
        ip6_flow_hdr(ipv6h, dsfield,
                     ip6_make_flowlabel(net, skb, fl6->flowlabel, true, fl6));
        ipv6h->hop_limit = hop_limit;
        ipv6h->nexthdr = proto;
        ipv6h->saddr = fl6->saddr;
        ipv6h->daddr = fl6->daddr;
        ip6tunnel_xmit(NULL, skb, dev, 0);
        return 0;
tx_err_link_failure:
        DEV_STATS_INC(dev, tx_carrier_errors);
        dst_link_failure(skb);
tx_err_dst_release:
        dst_release(dst);
        return err;
}
EXPORT_SYMBOL(ip6_tnl_xmit);

static inline int
ipxip6_tnl_xmit(struct sk_buff *skb, struct net_device *dev,
                u8 protocol)
{
        struct ip6_tnl *t = netdev_priv(dev);
        struct ipv6hdr *ipv6h;
        const struct iphdr  *iph;
        int encap_limit = -1;
        __u16 offset;
        struct flowi6 fl6;
        __u8 dsfield, orig_dsfield;
        __u32 mtu;
        u8 tproto;
        int err;

        tproto = READ_ONCE(t->parms.proto);
        if (tproto != protocol && tproto != 0)
                return -1;

        if (t->parms.collect_md) {
                struct ip_tunnel_info *tun_info;
                const struct ip_tunnel_key *key;

                tun_info = skb_tunnel_info(skb);
                if (unlikely(!tun_info || !(tun_info->mode & IP_TUNNEL_INFO_TX) ||
                             ip_tunnel_info_af(tun_info) != AF_INET6))
                        return -1;
                key = &tun_info->key;
                memset(&fl6, 0, sizeof(fl6));
                fl6.flowi6_proto = protocol;
                fl6.saddr = key->u.ipv6.src;
                fl6.daddr = key->u.ipv6.dst;
                fl6.flowlabel = key->label;
                dsfield =  key->tos;
                switch (protocol) {
                case IPPROTO_IPIP:
                        iph = ip_hdr(skb);
                        orig_dsfield = ipv4_get_dsfield(iph);
                        break;
                case IPPROTO_IPV6:
                        ipv6h = ipv6_hdr(skb);
                        orig_dsfield = ipv6_get_dsfield(ipv6h);
                        break;
                default:
                        orig_dsfield = dsfield;
                        break;
                }
        } else {
                if (!(t->parms.flags & IP6_TNL_F_IGN_ENCAP_LIMIT))
                        encap_limit = t->parms.encap_limit;
                if (protocol == IPPROTO_IPV6) {
                        offset = ip6_tnl_parse_tlv_enc_lim(skb,
                                                skb_network_header(skb));
                        /* ip6_tnl_parse_tlv_enc_lim() might have
                         * reallocated skb->head
                         */
                        if (offset > 0) {
                                struct ipv6_tlv_tnl_enc_lim *tel;

                                tel = (void *)&skb_network_header(skb)[offset];
                                if (tel->encap_limit == 0) {
                                        icmpv6_ndo_send(skb, ICMPV6_PARAMPROB,
                                                        ICMPV6_HDR_FIELD, offset + 2);
                                        return -1;
                                }
                                encap_limit = tel->encap_limit - 1;
                        }
                }

                memcpy(&fl6, &t->fl.u.ip6, sizeof(fl6));
                fl6.flowi6_proto = protocol;

                if (t->parms.flags & IP6_TNL_F_USE_ORIG_FWMARK)
                        fl6.flowi6_mark = skb->mark;
                else
                        fl6.flowi6_mark = t->parms.fwmark;
                switch (protocol) {
                case IPPROTO_IPIP:
                        iph = ip_hdr(skb);
                        orig_dsfield = ipv4_get_dsfield(iph);
                        if (t->parms.flags & IP6_TNL_F_USE_ORIG_TCLASS)
                                dsfield = orig_dsfield;
                        else
                                dsfield = ip6_tclass(t->parms.flowinfo);
                        break;
                case IPPROTO_IPV6:
                        ipv6h = ipv6_hdr(skb);
                        orig_dsfield = ipv6_get_dsfield(ipv6h);
                        if (t->parms.flags & IP6_TNL_F_USE_ORIG_TCLASS)
                                dsfield = orig_dsfield;
                        else
                                dsfield = ip6_tclass(t->parms.flowinfo);
                        if (t->parms.flags & IP6_TNL_F_USE_ORIG_FLOWLABEL)
                                fl6.flowlabel |= ip6_flowlabel(ipv6h);
                        break;
                default:
                        orig_dsfield = dsfield = ip6_tclass(t->parms.flowinfo);
                        break;
                }
        }

        fl6.flowi6_uid = sock_net_uid(dev_net(dev), NULL);
        dsfield = INET_ECN_encapsulate(dsfield, orig_dsfield);

        if (iptunnel_handle_offloads(skb, SKB_GSO_IPXIP6))
                return -1;

        skb_set_inner_ipproto(skb, protocol);

        err = ip6_tnl_xmit(skb, dev, dsfield, &fl6, encap_limit, &mtu,
                           protocol);
        if (err != 0) {
                /* XXX: send ICMP error even if DF is not set. */
                if (err == -EMSGSIZE)
                        switch (protocol) {
                        case IPPROTO_IPIP:
                                icmp_ndo_send(skb, ICMP_DEST_UNREACH,
                                              ICMP_FRAG_NEEDED, htonl(mtu));
                                break;
                        case IPPROTO_IPV6:
                                icmpv6_ndo_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
                                break;
                        default:
                                break;
                        }
                return -1;
        }

        return 0;
}

static netdev_tx_t
ip6_tnl_start_xmit(struct sk_buff *skb, struct net_device *dev)
{
        struct ip6_tnl *t = netdev_priv(dev);
        u8 ipproto;
        int ret;

        if (!pskb_inet_may_pull(skb))
                goto tx_err;

        switch (skb->protocol) {
        case htons(ETH_P_IP):
                ipproto = IPPROTO_IPIP;
                break;
        case htons(ETH_P_IPV6):
                if (ip6_tnl_addr_conflict(t, ipv6_hdr(skb)))
                        goto tx_err;
                ipproto = IPPROTO_IPV6;
                break;
        case htons(ETH_P_MPLS_UC):
                ipproto = IPPROTO_MPLS;
                break;
        default:
                goto tx_err;
        }

        ret = ipxip6_tnl_xmit(skb, dev, ipproto);
        if (ret < 0)
                goto tx_err;

        return NETDEV_TX_OK;

tx_err:
        DEV_STATS_INC(dev, tx_errors);
        DEV_STATS_INC(dev, tx_dropped);
        kfree_skb(skb);
        return NETDEV_TX_OK;
}

static void ip6_tnl_link_config(struct ip6_tnl *t)
{
        struct net_device *dev = t->dev;
        struct net_device *tdev = NULL;
        struct __ip6_tnl_parm *p = &t->parms;
        struct flowi6 *fl6 = &t->fl.u.ip6;
        int t_hlen;
        int mtu;

        __dev_addr_set(dev, &p->laddr, sizeof(struct in6_addr));
        memcpy(dev->broadcast, &p->raddr, sizeof(struct in6_addr));

        /* Set up flowi template */
        fl6->saddr = p->laddr;
        fl6->daddr = p->raddr;
        fl6->flowi6_oif = p->link;
        fl6->flowlabel = 0;

        if (!(p->flags&IP6_TNL_F_USE_ORIG_TCLASS))
                fl6->flowlabel |= IPV6_TCLASS_MASK & p->flowinfo;
        if (!(p->flags&IP6_TNL_F_USE_ORIG_FLOWLABEL))
                fl6->flowlabel |= IPV6_FLOWLABEL_MASK & p->flowinfo;

        p->flags &= ~(IP6_TNL_F_CAP_XMIT|IP6_TNL_F_CAP_RCV|IP6_TNL_F_CAP_PER_PACKET);
        p->flags |= ip6_tnl_get_cap(t, &p->laddr, &p->raddr);

        if (p->flags&IP6_TNL_F_CAP_XMIT && p->flags&IP6_TNL_F_CAP_RCV)
                dev->flags |= IFF_POINTOPOINT;
        else
                dev->flags &= ~IFF_POINTOPOINT;

        t->tun_hlen = 0;
        t->hlen = t->encap_hlen + t->tun_hlen;
        t_hlen = t->hlen + sizeof(struct ipv6hdr);

        if (p->flags & IP6_TNL_F_CAP_XMIT) {
                int strict = (ipv6_addr_type(&p->raddr) &
                              (IPV6_ADDR_MULTICAST|IPV6_ADDR_LINKLOCAL));

                struct rt6_info *rt = rt6_lookup(t->net,
                                                 &p->raddr, &p->laddr,
                                                 p->link, NULL, strict);
                if (rt) {
                        tdev = rt->dst.dev;
                        ip6_rt_put(rt);
                }

                if (!tdev && p->link)
                        tdev = __dev_get_by_index(t->net, p->link);

                if (tdev) {
                        unsigned int headroom;

                        headroom = tdev->hard_header_len + tdev->needed_headroom;
                        headroom += t_hlen;
                        dev->needed_headroom = ip_tunnel_limit_headroom(headroom);
                        mtu = min_t(unsigned int, tdev->mtu, IP6_MAX_MTU);

                        mtu = mtu - t_hlen;
                        if (!(t->parms.flags & IP6_TNL_F_IGN_ENCAP_LIMIT))
                                mtu -= 8;

                        if (mtu < IPV6_MIN_MTU)
                                mtu = IPV6_MIN_MTU;
                        WRITE_ONCE(dev->mtu, mtu);
                }
        }
}

/**
 * ip6_tnl_change - update the tunnel parameters
 *   @t: tunnel to be changed
 *   @p: tunnel configuration parameters
 *
 * Description:
 *   ip6_tnl_change() updates the tunnel parameters
 **/

static void
ip6_tnl_change(struct ip6_tnl *t, const struct __ip6_tnl_parm *p)
{
        t->parms.laddr = p->laddr;
        t->parms.raddr = p->raddr;
        t->parms.flags = p->flags;
        t->parms.hop_limit = p->hop_limit;
        t->parms.encap_limit = p->encap_limit;
        t->parms.flowinfo = p->flowinfo;
        t->parms.link = p->link;
        t->parms.proto = p->proto;
        t->parms.fwmark = p->fwmark;
        dst_cache_reset(&t->dst_cache);
        ip6_tnl_link_config(t);
}

static void ip6_tnl_update(struct ip6_tnl *t, struct __ip6_tnl_parm *p)
{
        struct net *net = t->net;
        struct ip6_tnl_net *ip6n = net_generic(net, ip6_tnl_net_id);

        ip6_tnl_unlink(ip6n, t);
        synchronize_net();
        ip6_tnl_change(t, p);
        ip6_tnl_link(ip6n, t);
        netdev_state_change(t->dev);
}

static int ip6_tnl0_update(struct ip6_tnl *t, struct __ip6_tnl_parm *p,
                           bool strict)
{
        /* For the default ip6tnl0 device, allow changing only the protocol
         * (the IP6_TNL_F_CAP_PER_PACKET flag is set on ip6tnl0, and all other
         * parameters are 0).
         */
        if (strict &&
            (!ipv6_addr_any(&p->laddr) || !ipv6_addr_any(&p->raddr) ||
             p->flags != t->parms.flags || p->hop_limit || p->encap_limit ||
             p->flowinfo || p->link || p->fwmark || p->collect_md))
                return -EINVAL;

        t->parms.proto = p->proto;
        netdev_state_change(t->dev);
        return 0;
}

static void
ip6_tnl_parm_from_user(struct __ip6_tnl_parm *p, const struct ip6_tnl_parm *u)
{
        p->laddr = u->laddr;
        p->raddr = u->raddr;
        p->flags = u->flags;
        p->hop_limit = u->hop_limit;
        p->encap_limit = u->encap_limit;
        p->flowinfo = u->flowinfo;
        p->link = u->link;
        p->proto = u->proto;
        memcpy(p->name, u->name, sizeof(u->name));
}

static void
ip6_tnl_parm_to_user(struct ip6_tnl_parm *u, const struct __ip6_tnl_parm *p)
{
        u->laddr = p->laddr;
        u->raddr = p->raddr;
        u->flags = p->flags;
        u->hop_limit = p->hop_limit;
        u->encap_limit = p->encap_limit;
        u->flowinfo = p->flowinfo;
        u->link = p->link;
        u->proto = p->proto;
        memcpy(u->name, p->name, sizeof(u->name));
}

/**
 * ip6_tnl_siocdevprivate - configure ipv6 tunnels from userspace
 *   @dev: virtual device associated with tunnel
 *   @ifr: unused
 *   @data: parameters passed from userspace
 *   @cmd: command to be performed
 *
 * Description:
 *   ip6_tnl_ioctl() is used for managing IPv6 tunnels
 *   from userspace.
 *
 *   The possible commands are the following:
 *     %SIOCGETTUNNEL: get tunnel parameters for device
 *     %SIOCADDTUNNEL: add tunnel matching given tunnel parameters
 *     %SIOCCHGTUNNEL: change tunnel parameters to those given
 *     %SIOCDELTUNNEL: delete tunnel
 *
 *   The fallback device "ip6tnl0", created during module
 *   initialization, can be used for creating other tunnel devices.
 *
 * Return:
 *   0 on success,
 *   %-EFAULT if unable to copy data to or from userspace,
 *   %-EPERM if current process hasn't %CAP_NET_ADMIN set
 *   %-EINVAL if passed tunnel parameters are invalid,
 *   %-EEXIST if changing a tunnel's parameters would cause a conflict
 *   %-ENODEV if attempting to change or delete a nonexisting device
 **/

static int
ip6_tnl_siocdevprivate(struct net_device *dev, struct ifreq *ifr,
                       void __user *data, int cmd)
{
        int err = 0;
        struct ip6_tnl_parm p;
        struct __ip6_tnl_parm p1;
        struct ip6_tnl *t = netdev_priv(dev);
        struct net *net = t->net;
        struct ip6_tnl_net *ip6n = net_generic(net, ip6_tnl_net_id);

        memset(&p1, 0, sizeof(p1));

        switch (cmd) {
        case SIOCGETTUNNEL:
                if (dev == ip6n->fb_tnl_dev) {
                        if (copy_from_user(&p, data, sizeof(p))) {
                                err = -EFAULT;
                                break;
                        }
                        ip6_tnl_parm_from_user(&p1, &p);
                        t = ip6_tnl_locate(net, &p1, 0);
                        if (IS_ERR(t))
                                t = netdev_priv(dev);
                } else {
                        memset(&p, 0, sizeof(p));
                }
                ip6_tnl_parm_to_user(&p, &t->parms);
                if (copy_to_user(data, &p, sizeof(p)))
                        err = -EFAULT;
                break;
        case SIOCADDTUNNEL:
        case SIOCCHGTUNNEL:
                err = -EPERM;
                if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
                        break;
                err = -EFAULT;
                if (copy_from_user(&p, data, sizeof(p)))
                        break;
                err = -EINVAL;
                if (p.proto != IPPROTO_IPV6 && p.proto != IPPROTO_IPIP &&
                    p.proto != 0)
                        break;
                ip6_tnl_parm_from_user(&p1, &p);
                t = ip6_tnl_locate(net, &p1, cmd == SIOCADDTUNNEL);
                if (cmd == SIOCCHGTUNNEL) {
                        if (!IS_ERR(t)) {
                                if (t->dev != dev) {
                                        err = -EEXIST;
                                        break;
                                }
                        } else
                                t = netdev_priv(dev);
                        if (dev == ip6n->fb_tnl_dev)
                                ip6_tnl0_update(t, &p1, false);
                        else
                                ip6_tnl_update(t, &p1);
                }
                if (!IS_ERR(t)) {
                        err = 0;
                        ip6_tnl_parm_to_user(&p, &t->parms);
                        if (copy_to_user(data, &p, sizeof(p)))
                                err = -EFAULT;

                } else {
                        err = PTR_ERR(t);
                }
                break;
        case SIOCDELTUNNEL:
                err = -EPERM;
                if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
                        break;

                if (dev == ip6n->fb_tnl_dev) {
                        err = -EFAULT;
                        if (copy_from_user(&p, data, sizeof(p)))
                                break;
                        err = -ENOENT;
                        ip6_tnl_parm_from_user(&p1, &p);
                        t = ip6_tnl_locate(net, &p1, 0);
                        if (IS_ERR(t))
                                break;
                        err = -EPERM;
                        if (t->dev == ip6n->fb_tnl_dev)
                                break;
                        dev = t->dev;
                }
                err = 0;
                unregister_netdevice(dev);
                break;
        default:
                err = -EINVAL;
        }
        return err;
}

/**
 * ip6_tnl_change_mtu - change mtu manually for tunnel device
 *   @dev: virtual device associated with tunnel
 *   @new_mtu: the new mtu
 *
 * Return:
 *   0 on success,
 *   %-EINVAL if mtu too small
 **/

int ip6_tnl_change_mtu(struct net_device *dev, int new_mtu)
{
        struct ip6_tnl *tnl = netdev_priv(dev);
        int t_hlen;

        t_hlen = tnl->hlen + sizeof(struct ipv6hdr);
        if (tnl->parms.proto == IPPROTO_IPV6) {
                if (new_mtu < IPV6_MIN_MTU)
                        return -EINVAL;
        } else {
                if (new_mtu < ETH_MIN_MTU)
                        return -EINVAL;
        }
        if (tnl->parms.proto == IPPROTO_IPV6 || tnl->parms.proto == 0) {
                if (new_mtu > IP6_MAX_MTU - dev->hard_header_len - t_hlen)
                        return -EINVAL;
        } else {
                if (new_mtu > IP_MAX_MTU - dev->hard_header_len - t_hlen)
                        return -EINVAL;
        }
        WRITE_ONCE(dev->mtu, new_mtu);
        return 0;
}
EXPORT_SYMBOL(ip6_tnl_change_mtu);

int ip6_tnl_get_iflink(const struct net_device *dev)
{
        struct ip6_tnl *t = netdev_priv(dev);

        return READ_ONCE(t->parms.link);
}
EXPORT_SYMBOL(ip6_tnl_get_iflink);

int ip6_tnl_encap_add_ops(const struct ip6_tnl_encap_ops *ops,
                          unsigned int num)
{
        if (num >= MAX_IPTUN_ENCAP_OPS)
                return -ERANGE;

        return !cmpxchg((const struct ip6_tnl_encap_ops **)
                        &ip6tun_encaps[num],
                        NULL, ops) ? 0 : -1;
}
EXPORT_SYMBOL(ip6_tnl_encap_add_ops);

int ip6_tnl_encap_del_ops(const struct ip6_tnl_encap_ops *ops,
                          unsigned int num)
{
        int ret;

        if (num >= MAX_IPTUN_ENCAP_OPS)
                return -ERANGE;

        ret = (cmpxchg((const struct ip6_tnl_encap_ops **)
                       &ip6tun_encaps[num],
                       ops, NULL) == ops) ? 0 : -1;

        synchronize_net();

        return ret;
}
EXPORT_SYMBOL(ip6_tnl_encap_del_ops);

int ip6_tnl_encap_setup(struct ip6_tnl *t,
                        struct ip_tunnel_encap *ipencap)
{
        int hlen;

        memset(&t->encap, 0, sizeof(t->encap));

        hlen = ip6_encap_hlen(ipencap);
        if (hlen < 0)
                return hlen;

        t->encap.type = ipencap->type;
        t->encap.sport = ipencap->sport;
        t->encap.dport = ipencap->dport;
        t->encap.flags = ipencap->flags;

        t->encap_hlen = hlen;
        t->hlen = t->encap_hlen + t->tun_hlen;

        return 0;
}
EXPORT_SYMBOL_GPL(ip6_tnl_encap_setup);

static int ip6_tnl_fill_forward_path(struct net_device_path_ctx *ctx,
                                     struct net_device_path *path)
{
        struct ip6_tnl *t = netdev_priv(ctx->dev);
        struct dst_entry *dst;
        struct flowi6 fl6;
        int err;

        if (ctx->ether_type != cpu_to_be16(ETH_P_IPV6))
                return -EOPNOTSUPP;

        if (t->parms.flags & (IP6_TNL_F_USE_ORIG_TCLASS |
                              IP6_TNL_F_USE_ORIG_FLOWLABEL |
                              IP6_TNL_F_USE_ORIG_FWMARK))
                return -EOPNOTSUPP;

        if (t->parms.collect_md)
                return -EOPNOTSUPP;

        if (!(t->parms.flags & IP6_TNL_F_IGN_ENCAP_LIMIT))
                return -EOPNOTSUPP;

        memcpy(&fl6, &t->fl.u.ip6, sizeof(fl6));
        fl6.flowi6_mark = t->parms.fwmark;
        fl6.flowi6_proto = 0;

        dst = ip6_route_output(dev_net(ctx->dev), NULL, &fl6);
        if (!dst->error) {
                path->type = DEV_PATH_TUN;
                path->tun.src_v6 = fl6.saddr;
                path->tun.dst_v6 = fl6.daddr;
                path->tun.inner_proto = IPPROTO_IPV6;
                path->tun.dst = dst;
                path->dev = ctx->dev;
                ctx->dev = dst->dev;
        }

        err = dst->error;
        if (err)
                dst_release(dst);

        return err;
}

static const struct net_device_ops ip6_tnl_netdev_ops = {
        .ndo_init        = ip6_tnl_dev_init,
        .ndo_uninit        = ip6_tnl_dev_uninit,
        .ndo_start_xmit = ip6_tnl_start_xmit,
        .ndo_siocdevprivate = ip6_tnl_siocdevprivate,
        .ndo_change_mtu = ip6_tnl_change_mtu,
        .ndo_get_stats64 = dev_get_tstats64,
        .ndo_get_iflink = ip6_tnl_get_iflink,
        .ndo_fill_forward_path = ip6_tnl_fill_forward_path,
};

#define IPXIPX_FEATURES (NETIF_F_SG |                \
                         NETIF_F_FRAGLIST |        \
                         NETIF_F_HIGHDMA |        \
                         NETIF_F_GSO_SOFTWARE |        \
                         NETIF_F_HW_CSUM)

/**
 * ip6_tnl_dev_setup - setup virtual tunnel device
 *   @dev: virtual device associated with tunnel
 *
 * Description:
 *   Initialize function pointers and device parameters
 **/

static void ip6_tnl_dev_setup(struct net_device *dev)
{
        dev->netdev_ops = &ip6_tnl_netdev_ops;
        dev->header_ops = &ip_tunnel_header_ops;
        dev->needs_free_netdev = true;
        dev->priv_destructor = ip6_dev_free;

        dev->type = ARPHRD_TUNNEL6;
        dev->flags |= IFF_NOARP;
        dev->addr_len = sizeof(struct in6_addr);
        dev->lltx = true;
        dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
        netif_keep_dst(dev);

        dev->features                |= IPXIPX_FEATURES;
        dev->hw_features        |= IPXIPX_FEATURES;

        /* This perm addr will be used as interface identifier by IPv6 */
        dev->addr_assign_type = NET_ADDR_RANDOM;
        eth_random_addr(dev->perm_addr);
}


/**
 * ip6_tnl_dev_init_gen - general initializer for all tunnel devices
 *   @dev: virtual device associated with tunnel
 **/

static inline int
ip6_tnl_dev_init_gen(struct net_device *dev)
{
        struct ip6_tnl *t = netdev_priv(dev);
        int ret;
        int t_hlen;

        t->dev = dev;

        ret = dst_cache_init(&t->dst_cache, GFP_KERNEL);
        if (ret)
                return ret;

        ret = gro_cells_init(&t->gro_cells, dev);
        if (ret)
                goto destroy_dst;

        t->tun_hlen = 0;
        t->hlen = t->encap_hlen + t->tun_hlen;
        t_hlen = t->hlen + sizeof(struct ipv6hdr);

        dev->type = ARPHRD_TUNNEL6;
        dev->mtu = ETH_DATA_LEN - t_hlen;
        if (!(t->parms.flags & IP6_TNL_F_IGN_ENCAP_LIMIT))
                dev->mtu -= 8;
        dev->min_mtu = ETH_MIN_MTU;
        dev->max_mtu = IP6_MAX_MTU - dev->hard_header_len - t_hlen;

        netdev_hold(dev, &t->dev_tracker, GFP_KERNEL);
        netdev_lockdep_set_classes(dev);
        return 0;

destroy_dst:
        dst_cache_destroy(&t->dst_cache);

        return ret;
}

/**
 * ip6_tnl_dev_init - initializer for all non fallback tunnel devices
 *   @dev: virtual device associated with tunnel
 **/

static int ip6_tnl_dev_init(struct net_device *dev)
{
        struct ip6_tnl *t = netdev_priv(dev);
        int err = ip6_tnl_dev_init_gen(dev);

        if (err)
                return err;
        ip6_tnl_link_config(t);
        if (t->parms.collect_md)
                netif_keep_dst(dev);
        return 0;
}

/**
 * ip6_fb_tnl_dev_init - initializer for fallback tunnel device
 *   @dev: fallback device
 *
 * Return: 0
 **/

static int __net_init ip6_fb_tnl_dev_init(struct net_device *dev)
{
        struct ip6_tnl *t = netdev_priv(dev);
        struct net *net = dev_net(dev);
        struct ip6_tnl_net *ip6n = net_generic(net, ip6_tnl_net_id);

        t->net = net;
        t->parms.proto = IPPROTO_IPV6;

        rcu_assign_pointer(ip6n->tnls_wc[0], t);
        return 0;
}

static int ip6_tnl_validate(struct nlattr *tb[], struct nlattr *data[],
                            struct netlink_ext_ack *extack)
{
        u8 proto;

        if (!data || !data[IFLA_IPTUN_PROTO])
                return 0;

        proto = nla_get_u8(data[IFLA_IPTUN_PROTO]);
        if (proto != IPPROTO_IPV6 &&
            proto != IPPROTO_IPIP &&
            proto != 0)
                return -EINVAL;

        return 0;
}

static void ip6_tnl_netlink_parms(struct nlattr *data[],
                                  struct __ip6_tnl_parm *parms)
{
        memset(parms, 0, sizeof(*parms));

        if (!data)
                return;

        if (data[IFLA_IPTUN_LINK])
                parms->link = nla_get_u32(data[IFLA_IPTUN_LINK]);

        if (data[IFLA_IPTUN_LOCAL])
                parms->laddr = nla_get_in6_addr(data[IFLA_IPTUN_LOCAL]);

        if (data[IFLA_IPTUN_REMOTE])
                parms->raddr = nla_get_in6_addr(data[IFLA_IPTUN_REMOTE]);

        if (data[IFLA_IPTUN_TTL])
                parms->hop_limit = nla_get_u8(data[IFLA_IPTUN_TTL]);

        if (data[IFLA_IPTUN_ENCAP_LIMIT])
                parms->encap_limit = nla_get_u8(data[IFLA_IPTUN_ENCAP_LIMIT]);

        if (data[IFLA_IPTUN_FLOWINFO])
                parms->flowinfo = nla_get_be32(data[IFLA_IPTUN_FLOWINFO]);

        if (data[IFLA_IPTUN_FLAGS])
                parms->flags = nla_get_u32(data[IFLA_IPTUN_FLAGS]);

        if (data[IFLA_IPTUN_PROTO])
                parms->proto = nla_get_u8(data[IFLA_IPTUN_PROTO]);

        if (data[IFLA_IPTUN_COLLECT_METADATA])
                parms->collect_md = true;

        if (data[IFLA_IPTUN_FWMARK])
                parms->fwmark = nla_get_u32(data[IFLA_IPTUN_FWMARK]);
}

static int ip6_tnl_newlink(struct net_device *dev,
                           struct rtnl_newlink_params *params,
                           struct netlink_ext_ack *extack)
{
        struct nlattr **data = params->data;
        struct nlattr **tb = params->tb;
        struct ip_tunnel_encap ipencap;
        struct ip6_tnl_net *ip6n;
        struct ip6_tnl *nt, *t;
        struct net *net;
        int err;

        net = params->link_net ? : dev_net(dev);
        ip6n = net_generic(net, ip6_tnl_net_id);
        nt = netdev_priv(dev);
        nt->net = net;

        if (ip_tunnel_netlink_encap_parms(data, &ipencap)) {
                err = ip6_tnl_encap_setup(nt, &ipencap);
                if (err < 0)
                        return err;
        }

        ip6_tnl_netlink_parms(data, &nt->parms);

        if (nt->parms.collect_md) {
                if (rtnl_dereference(ip6n->collect_md_tun))
                        return -EEXIST;
        } else {
                t = ip6_tnl_locate(net, &nt->parms, 0);
                if (!IS_ERR(t))
                        return -EEXIST;
        }

        err = ip6_tnl_create2(dev);
        if (!err && tb[IFLA_MTU])
                ip6_tnl_change_mtu(dev, nla_get_u32(tb[IFLA_MTU]));

        return err;
}

static int ip6_tnl_changelink(struct net_device *dev, struct nlattr *tb[],
                              struct nlattr *data[],
                              struct netlink_ext_ack *extack)
{
        struct ip6_tnl *t = netdev_priv(dev);
        struct __ip6_tnl_parm p;
        struct net *net = t->net;
        struct ip6_tnl_net *ip6n = net_generic(net, ip6_tnl_net_id);
        struct ip_tunnel_encap ipencap;

        if (!rtnl_dev_link_net_capable(dev, net))
                return -EPERM;

        if (dev == ip6n->fb_tnl_dev) {
                if (ip_tunnel_netlink_encap_parms(data, &ipencap)) {
                        /* iproute2 always sets TUNNEL_ENCAP_FLAG_CSUM6, so
                         * let's ignore this flag.
                         */
                        ipencap.flags &= ~TUNNEL_ENCAP_FLAG_CSUM6;
                        if (memchr_inv(&ipencap, 0, sizeof(ipencap))) {
                                NL_SET_ERR_MSG(extack,
                                               "Only protocol can be changed for fallback tunnel, not encap params");
                                return -EINVAL;
                        }
                }

                ip6_tnl_netlink_parms(data, &p);
                if (ip6_tnl0_update(t, &p, true) < 0) {
                        NL_SET_ERR_MSG(extack,
                                       "Only protocol can be changed for fallback tunnel");
                        return -EINVAL;
                }

                return 0;
        }

        if (ip_tunnel_netlink_encap_parms(data, &ipencap)) {
                int err = ip6_tnl_encap_setup(t, &ipencap);

                if (err < 0)
                        return err;
        }
        ip6_tnl_netlink_parms(data, &p);
        if (p.collect_md)
                return -EINVAL;

        t = ip6_tnl_locate(net, &p, 0);
        if (!IS_ERR(t)) {
                if (t->dev != dev)
                        return -EEXIST;
        } else
                t = netdev_priv(dev);

        ip6_tnl_update(t, &p);
        return 0;
}

static void ip6_tnl_dellink(struct net_device *dev, struct list_head *head)
{
        struct net *net = dev_net(dev);
        struct ip6_tnl_net *ip6n = net_generic(net, ip6_tnl_net_id);

        if (dev != ip6n->fb_tnl_dev)
                unregister_netdevice_queue(dev, head);
}

static size_t ip6_tnl_get_size(const struct net_device *dev)
{
        return
                /* IFLA_IPTUN_LINK */
                nla_total_size(4) +
                /* IFLA_IPTUN_LOCAL */
                nla_total_size(sizeof(struct in6_addr)) +
                /* IFLA_IPTUN_REMOTE */
                nla_total_size(sizeof(struct in6_addr)) +
                /* IFLA_IPTUN_TTL */
                nla_total_size(1) +
                /* IFLA_IPTUN_ENCAP_LIMIT */
                nla_total_size(1) +
                /* IFLA_IPTUN_FLOWINFO */
                nla_total_size(4) +
                /* IFLA_IPTUN_FLAGS */
                nla_total_size(4) +
                /* IFLA_IPTUN_PROTO */
                nla_total_size(1) +
                /* IFLA_IPTUN_ENCAP_TYPE */
                nla_total_size(2) +
                /* IFLA_IPTUN_ENCAP_FLAGS */
                nla_total_size(2) +
                /* IFLA_IPTUN_ENCAP_SPORT */
                nla_total_size(2) +
                /* IFLA_IPTUN_ENCAP_DPORT */
                nla_total_size(2) +
                /* IFLA_IPTUN_COLLECT_METADATA */
                nla_total_size(0) +
                /* IFLA_IPTUN_FWMARK */
                nla_total_size(4) +
                0;
}

static int ip6_tnl_fill_info(struct sk_buff *skb, const struct net_device *dev)
{
        struct ip6_tnl *tunnel = netdev_priv(dev);
        struct __ip6_tnl_parm *parm = &tunnel->parms;

        if (nla_put_u32(skb, IFLA_IPTUN_LINK, parm->link) ||
            nla_put_in6_addr(skb, IFLA_IPTUN_LOCAL, &parm->laddr) ||
            nla_put_in6_addr(skb, IFLA_IPTUN_REMOTE, &parm->raddr) ||
            nla_put_u8(skb, IFLA_IPTUN_TTL, parm->hop_limit) ||
            nla_put_u8(skb, IFLA_IPTUN_ENCAP_LIMIT, parm->encap_limit) ||
            nla_put_be32(skb, IFLA_IPTUN_FLOWINFO, parm->flowinfo) ||
            nla_put_u32(skb, IFLA_IPTUN_FLAGS, parm->flags) ||
            nla_put_u8(skb, IFLA_IPTUN_PROTO, parm->proto) ||
            nla_put_u32(skb, IFLA_IPTUN_FWMARK, parm->fwmark))
                goto nla_put_failure;

        if (nla_put_u16(skb, IFLA_IPTUN_ENCAP_TYPE, tunnel->encap.type) ||
            nla_put_be16(skb, IFLA_IPTUN_ENCAP_SPORT, tunnel->encap.sport) ||
            nla_put_be16(skb, IFLA_IPTUN_ENCAP_DPORT, tunnel->encap.dport) ||
            nla_put_u16(skb, IFLA_IPTUN_ENCAP_FLAGS, tunnel->encap.flags))
                goto nla_put_failure;

        if (parm->collect_md)
                if (nla_put_flag(skb, IFLA_IPTUN_COLLECT_METADATA))
                        goto nla_put_failure;

        return 0;

nla_put_failure:
        return -EMSGSIZE;
}

struct net *ip6_tnl_get_link_net(const struct net_device *dev)
{
        struct ip6_tnl *tunnel = netdev_priv(dev);

        return READ_ONCE(tunnel->net);
}
EXPORT_SYMBOL(ip6_tnl_get_link_net);

static const struct nla_policy ip6_tnl_policy[IFLA_IPTUN_MAX + 1] = {
        [IFLA_IPTUN_LINK]                = { .type = NLA_U32 },
        [IFLA_IPTUN_LOCAL]                = { .len = sizeof(struct in6_addr) },
        [IFLA_IPTUN_REMOTE]                = { .len = sizeof(struct in6_addr) },
        [IFLA_IPTUN_TTL]                = { .type = NLA_U8 },
        [IFLA_IPTUN_ENCAP_LIMIT]        = { .type = NLA_U8 },
        [IFLA_IPTUN_FLOWINFO]                = { .type = NLA_U32 },
        [IFLA_IPTUN_FLAGS]                = { .type = NLA_U32 },
        [IFLA_IPTUN_PROTO]                = { .type = NLA_U8 },
        [IFLA_IPTUN_ENCAP_TYPE]                = { .type = NLA_U16 },
        [IFLA_IPTUN_ENCAP_FLAGS]        = { .type = NLA_U16 },
        [IFLA_IPTUN_ENCAP_SPORT]        = { .type = NLA_U16 },
        [IFLA_IPTUN_ENCAP_DPORT]        = { .type = NLA_U16 },
        [IFLA_IPTUN_COLLECT_METADATA]        = { .type = NLA_FLAG },
        [IFLA_IPTUN_FWMARK]                = { .type = NLA_U32 },
};

static struct rtnl_link_ops ip6_link_ops __read_mostly = {
        .kind                = "ip6tnl",
        .maxtype        = IFLA_IPTUN_MAX,
        .policy                = ip6_tnl_policy,
        .priv_size        = sizeof(struct ip6_tnl),
        .setup                = ip6_tnl_dev_setup,
        .validate        = ip6_tnl_validate,
        .newlink        = ip6_tnl_newlink,
        .changelink        = ip6_tnl_changelink,
        .dellink        = ip6_tnl_dellink,
        .get_size        = ip6_tnl_get_size,
        .fill_info        = ip6_tnl_fill_info,
        .get_link_net        = ip6_tnl_get_link_net,
};

static struct xfrm6_tunnel ip4ip6_handler __read_mostly = {
        .handler        = ip4ip6_rcv,
        .err_handler        = ip4ip6_err,
        .priority        =        1,
};

static struct xfrm6_tunnel ip6ip6_handler __read_mostly = {
        .handler        = ip6ip6_rcv,
        .err_handler        = ip6ip6_err,
        .priority        =        1,
};

static struct xfrm6_tunnel mplsip6_handler __read_mostly = {
        .handler        = mplsip6_rcv,
        .err_handler        = mplsip6_err,
        .priority        =        1,
};

static void __net_exit ip6_tnl_exit_rtnl_net(struct net *net, struct list_head *list)
{
        struct ip6_tnl_net *ip6n = net_generic(net, ip6_tnl_net_id);
        struct net_device *dev, *aux;
        int h;
        struct ip6_tnl *t;

        for_each_netdev_safe(net, dev, aux)
                if (dev->rtnl_link_ops == &ip6_link_ops)
                        unregister_netdevice_queue(dev, list);

        for (h = 0; h < IP6_TUNNEL_HASH_SIZE; h++) {
                t = rtnl_net_dereference(net, ip6n->tnls_r_l[h]);
                while (t) {
                        /* If dev is in the same netns, it has already
                         * been added to the list by the previous loop.
                         */
                        if (!net_eq(dev_net(t->dev), net))
                                unregister_netdevice_queue(t->dev, list);

                        t = rtnl_net_dereference(net, t->next);
                }
        }

        t = rtnl_net_dereference(net, ip6n->tnls_wc[0]);
        while (t) {
                /* If dev is in the same netns, it has already
                 * been added to the list by the previous loop.
                 */
                if (!net_eq(dev_net(t->dev), net))
                        unregister_netdevice_queue(t->dev, list);

                t = rtnl_net_dereference(net, t->next);
        }
}

static int __net_init ip6_tnl_init_net(struct net *net)
{
        struct ip6_tnl_net *ip6n = net_generic(net, ip6_tnl_net_id);
        struct ip6_tnl *t = NULL;
        int err;

        ip6n->tnls[0] = ip6n->tnls_wc;
        ip6n->tnls[1] = ip6n->tnls_r_l;

        if (!net_has_fallback_tunnels(net))
                return 0;
        err = -ENOMEM;
        ip6n->fb_tnl_dev = alloc_netdev(sizeof(struct ip6_tnl), "ip6tnl0",
                                        NET_NAME_UNKNOWN, ip6_tnl_dev_setup);

        if (!ip6n->fb_tnl_dev)
                goto err_alloc_dev;
        dev_net_set(ip6n->fb_tnl_dev, net);
        ip6n->fb_tnl_dev->rtnl_link_ops = &ip6_link_ops;
        /* FB netdevice is special: we have one, and only one per netns.
         * Allowing to move it to another netns is clearly unsafe.
         */
        ip6n->fb_tnl_dev->netns_immutable = true;

        err = ip6_fb_tnl_dev_init(ip6n->fb_tnl_dev);
        if (err < 0)
                goto err_register;

        err = register_netdev(ip6n->fb_tnl_dev);
        if (err < 0)
                goto err_register;

        t = netdev_priv(ip6n->fb_tnl_dev);

        strcpy(t->parms.name, ip6n->fb_tnl_dev->name);
        return 0;

err_register:
        free_netdev(ip6n->fb_tnl_dev);
err_alloc_dev:
        return err;
}

static struct pernet_operations ip6_tnl_net_ops = {
        .init = ip6_tnl_init_net,
        .exit_rtnl = ip6_tnl_exit_rtnl_net,
        .id   = &ip6_tnl_net_id,
        .size = sizeof(struct ip6_tnl_net),
};

/**
 * ip6_tunnel_init - register protocol and reserve needed resources
 *
 * Return: 0 on success
 **/

static int __init ip6_tunnel_init(void)
{
        int  err;

        if (!ipv6_mod_enabled())
                return -EOPNOTSUPP;

        err = register_pernet_device(&ip6_tnl_net_ops);
        if (err < 0)
                goto out_pernet;

        err = xfrm6_tunnel_register(&ip4ip6_handler, AF_INET);
        if (err < 0) {
                pr_err("%s: can't register ip4ip6\n", __func__);
                goto out_ip4ip6;
        }

        err = xfrm6_tunnel_register(&ip6ip6_handler, AF_INET6);
        if (err < 0) {
                pr_err("%s: can't register ip6ip6\n", __func__);
                goto out_ip6ip6;
        }

        if (ip6_tnl_mpls_supported()) {
                err = xfrm6_tunnel_register(&mplsip6_handler, AF_MPLS);
                if (err < 0) {
                        pr_err("%s: can't register mplsip6\n", __func__);
                        goto out_mplsip6;
                }
        }

        err = rtnl_link_register(&ip6_link_ops);
        if (err < 0)
                goto rtnl_link_failed;

        return 0;

rtnl_link_failed:
        if (ip6_tnl_mpls_supported())
                xfrm6_tunnel_deregister(&mplsip6_handler, AF_MPLS);
out_mplsip6:
        xfrm6_tunnel_deregister(&ip6ip6_handler, AF_INET6);
out_ip6ip6:
        xfrm6_tunnel_deregister(&ip4ip6_handler, AF_INET);
out_ip4ip6:
        unregister_pernet_device(&ip6_tnl_net_ops);
out_pernet:
        return err;
}

/**
 * ip6_tunnel_cleanup - free resources and unregister protocol
 **/

static void __exit ip6_tunnel_cleanup(void)
{
        rtnl_link_unregister(&ip6_link_ops);
        if (xfrm6_tunnel_deregister(&ip4ip6_handler, AF_INET))
                pr_info("%s: can't deregister ip4ip6\n", __func__);

        if (xfrm6_tunnel_deregister(&ip6ip6_handler, AF_INET6))
                pr_info("%s: can't deregister ip6ip6\n", __func__);

        if (ip6_tnl_mpls_supported() &&
            xfrm6_tunnel_deregister(&mplsip6_handler, AF_MPLS))
                pr_info("%s: can't deregister mplsip6\n", __func__);
        unregister_pernet_device(&ip6_tnl_net_ops);
}

module_init(ip6_tunnel_init);
module_exit(ip6_tunnel_cleanup);


















    1 






    1 


    1 







    1 






    1 



    1 





    1 




    1 












    2 















    1 























    1 











    3 











    3 














    3 

















    3 















    3 














    1 














    1 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
// SPDX-License-Identifier: GPL-2.0-or-later

#include "linux/virtio_pci.h"
#include <linux/virtio_pci_legacy.h>
#include <linux/module.h>
#include <linux/pci.h>


/*
 * vp_legacy_probe: probe the legacy virtio pci device, note that the
 * caller is required to enable PCI device before calling this function.
 * @ldev: the legacy virtio-pci device
 *
 * Return 0 on succeed otherwise fail
 */
int vp_legacy_probe(struct virtio_pci_legacy_device *ldev)
{
        struct pci_dev *pci_dev = ldev->pci_dev;
        int rc;

        /* We only own devices >= 0x1000 and <= 0x103f: leave the rest. */
        if (pci_dev->device < 0x1000 || pci_dev->device > 0x103f)
                return -ENODEV;

        if (pci_dev->revision != VIRTIO_PCI_ABI_VERSION)
                return -ENODEV;

        rc = dma_set_mask(&pci_dev->dev, DMA_BIT_MASK(64));
        if (rc) {
                rc = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(32));
        } else {
                /*
                 * The virtio ring base address is expressed as a 32-bit PFN,
                 * with a page size of 1 << VIRTIO_PCI_QUEUE_ADDR_SHIFT.
                 */
                dma_set_coherent_mask(&pci_dev->dev,
                                DMA_BIT_MASK(32 + VIRTIO_PCI_QUEUE_ADDR_SHIFT));
        }

        if (rc)
                dev_warn(&pci_dev->dev, "Failed to enable 64-bit or 32-bit DMA.  Trying to continue, but this might not work.\n");

        rc = pci_request_region(pci_dev, 0, "virtio-pci-legacy");
        if (rc)
                return rc;

        ldev->ioaddr = pci_iomap(pci_dev, 0, 0);
        if (!ldev->ioaddr) {
                rc = -EIO;
                goto err_iomap;
        }

        ldev->isr = ldev->ioaddr + VIRTIO_PCI_ISR;

        ldev->id.vendor = pci_dev->subsystem_vendor;
        ldev->id.device = pci_dev->subsystem_device;

        return 0;
err_iomap:
        pci_release_region(pci_dev, 0);
        return rc;
}
EXPORT_SYMBOL_GPL(vp_legacy_probe);

/*
 * vp_legacy_probe: remove and cleanup the legacy virtio pci device
 * @ldev: the legacy virtio-pci device
 */
void vp_legacy_remove(struct virtio_pci_legacy_device *ldev)
{
        struct pci_dev *pci_dev = ldev->pci_dev;

        pci_iounmap(pci_dev, ldev->ioaddr);
        pci_release_region(pci_dev, 0);
}
EXPORT_SYMBOL_GPL(vp_legacy_remove);

/*
 * vp_legacy_get_features - get features from device
 * @ldev: the legacy virtio-pci device
 *
 * Returns the features read from the device
 */
u64 vp_legacy_get_features(struct virtio_pci_legacy_device *ldev)
{

        return ioread32(ldev->ioaddr + VIRTIO_PCI_HOST_FEATURES);
}
EXPORT_SYMBOL_GPL(vp_legacy_get_features);

/*
 * vp_legacy_get_driver_features - get driver features from device
 * @ldev: the legacy virtio-pci device
 *
 * Returns the driver features read from the device
 */
u64 vp_legacy_get_driver_features(struct virtio_pci_legacy_device *ldev)
{
        return ioread32(ldev->ioaddr + VIRTIO_PCI_GUEST_FEATURES);
}
EXPORT_SYMBOL_GPL(vp_legacy_get_driver_features);

/*
 * vp_legacy_set_features - set features to device
 * @ldev: the legacy virtio-pci device
 * @features: the features set to device
 */
void vp_legacy_set_features(struct virtio_pci_legacy_device *ldev,
                            u32 features)
{
        iowrite32(features, ldev->ioaddr + VIRTIO_PCI_GUEST_FEATURES);
}
EXPORT_SYMBOL_GPL(vp_legacy_set_features);

/*
 * vp_legacy_get_status - get the device status
 * @ldev: the legacy virtio-pci device
 *
 * Returns the status read from device
 */
u8 vp_legacy_get_status(struct virtio_pci_legacy_device *ldev)
{
        return ioread8(ldev->ioaddr + VIRTIO_PCI_STATUS);
}
EXPORT_SYMBOL_GPL(vp_legacy_get_status);

/*
 * vp_legacy_set_status - set status to device
 * @ldev: the legacy virtio-pci device
 * @status: the status set to device
 */
void vp_legacy_set_status(struct virtio_pci_legacy_device *ldev,
                                 u8 status)
{
        iowrite8(status, ldev->ioaddr + VIRTIO_PCI_STATUS);
}
EXPORT_SYMBOL_GPL(vp_legacy_set_status);

/*
 * vp_legacy_queue_vector - set the MSIX vector for a specific virtqueue
 * @ldev: the legacy virtio-pci device
 * @index: queue index
 * @vector: the queue vector
 *
 * Returns the queue vector read from the device
 */
u16 vp_legacy_queue_vector(struct virtio_pci_legacy_device *ldev,
                           u16 index, u16 vector)
{
        iowrite16(index, ldev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
        iowrite16(vector, ldev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
        /* Flush the write out to device */
        return ioread16(ldev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
}
EXPORT_SYMBOL_GPL(vp_legacy_queue_vector);

/*
 * vp_legacy_config_vector - set the vector for config interrupt
 * @ldev: the legacy virtio-pci device
 * @vector: the config vector
 *
 * Returns the config vector read from the device
 */
u16 vp_legacy_config_vector(struct virtio_pci_legacy_device *ldev,
                            u16 vector)
{
        /* Setup the vector used for configuration events */
        iowrite16(vector, ldev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
        /* Verify we had enough resources to assign the vector */
        /* Will also flush the write out to device */
        return ioread16(ldev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
}
EXPORT_SYMBOL_GPL(vp_legacy_config_vector);

/*
 * vp_legacy_set_queue_address - set the virtqueue address
 * @ldev: the legacy virtio-pci device
 * @index: the queue index
 * @queue_pfn: pfn of the virtqueue
 */
void vp_legacy_set_queue_address(struct virtio_pci_legacy_device *ldev,
                             u16 index, u32 queue_pfn)
{
        iowrite16(index, ldev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
        iowrite32(queue_pfn, ldev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
}
EXPORT_SYMBOL_GPL(vp_legacy_set_queue_address);

/*
 * vp_legacy_get_queue_enable - enable a virtqueue
 * @ldev: the legacy virtio-pci device
 * @index: the queue index
 *
 * Returns whether a virtqueue is enabled or not
 */
bool vp_legacy_get_queue_enable(struct virtio_pci_legacy_device *ldev,
                                u16 index)
{
        iowrite16(index, ldev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
        return ioread32(ldev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
}
EXPORT_SYMBOL_GPL(vp_legacy_get_queue_enable);

/*
 * vp_legacy_get_queue_size - get size for a virtqueue
 * @ldev: the legacy virtio-pci device
 * @index: the queue index
 *
 * Returns the size of the virtqueue
 */
u16 vp_legacy_get_queue_size(struct virtio_pci_legacy_device *ldev,
                             u16 index)
{
        iowrite16(index, ldev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
        return ioread16(ldev->ioaddr + VIRTIO_PCI_QUEUE_NUM);
}
EXPORT_SYMBOL_GPL(vp_legacy_get_queue_size);

MODULE_VERSION("0.1");
MODULE_DESCRIPTION("Legacy Virtio PCI Device");
MODULE_AUTHOR("Wu Zongyong <wuzongyong@linux.alibaba.com>");
MODULE_LICENSE("GPL");

















   30 








   30 
   30 


   30 
   10 

   30 
    1 



   30 







   36 



   36 
   11 

    1 
    1 


   36 



    3 
    3 


   35 

   35 






    6 

    4 



    6 
    6 



    4 


    4 


    4 






    3 



    3 


    6 














   38 






    2 
   38 











   38 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/file.h>
#include <linux/slab.h>
#include <linux/net.h>
#include <linux/io_uring.h>

#include "io_uring.h"
#include "notif.h"
#include "rsrc.h"

static const struct ubuf_info_ops io_ubuf_ops;

static void io_notif_tw_complete(struct io_tw_req tw_req, io_tw_token_t tw)
{
        struct io_kiocb *notif = tw_req.req;
        struct io_notif_data *nd = io_notif_to_data(notif);
        struct io_ring_ctx *ctx = notif->ctx;

        lockdep_assert_held(&ctx->uring_lock);

        do {
                notif = cmd_to_io_kiocb(nd);

                if (WARN_ON_ONCE(ctx != notif->ctx))
                        return;
                lockdep_assert(refcount_read(&nd->uarg.refcnt) == 0);

                if (unlikely(nd->zc_report) && (nd->zc_copied || !nd->zc_used))
                        notif->cqe.res |= IORING_NOTIF_USAGE_ZC_COPIED;

                if (nd->account_pages && notif->ctx->user) {
                        __io_unaccount_mem(notif->ctx->user, nd->account_pages);
                        nd->account_pages = 0;
                }

                nd = nd->next;
                io_req_task_complete((struct io_tw_req){notif}, tw);
        } while (nd);
}

void io_tx_ubuf_complete(struct sk_buff *skb, struct ubuf_info *uarg,
                         bool success)
{
        struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg);
        struct io_kiocb *notif = cmd_to_io_kiocb(nd);
        unsigned tw_flags;

        if (nd->zc_report) {
                if (success && !nd->zc_used && skb)
                        WRITE_ONCE(nd->zc_used, true);
                else if (!success && !nd->zc_copied)
                        WRITE_ONCE(nd->zc_copied, true);
        }

        if (!refcount_dec_and_test(&uarg->refcnt))
                return;

        if (nd->head != nd) {
                io_tx_ubuf_complete(skb, &nd->head->uarg, success);
                return;
        }

        tw_flags = nd->next ? 0 : IOU_F_TWQ_LAZY_WAKE;
        notif->io_task_work.func = io_notif_tw_complete;
        __io_req_task_work_add(notif, tw_flags);
}

static int io_link_skb(struct sk_buff *skb, struct ubuf_info *uarg)
{
        struct io_notif_data *nd, *prev_nd;
        struct io_kiocb *prev_notif, *notif;
        struct ubuf_info *prev_uarg = skb_zcopy(skb);

        nd = container_of(uarg, struct io_notif_data, uarg);
        notif = cmd_to_io_kiocb(nd);

        if (!prev_uarg) {
                net_zcopy_get(&nd->uarg);
                skb_zcopy_init(skb, &nd->uarg);
                return 0;
        }
        /* handle it separately as we can't link a notif to itself */
        if (unlikely(prev_uarg == &nd->uarg))
                return 0;
        /* we can't join two links together, just request a fresh skb */
        if (unlikely(nd->head != nd || nd->next))
                return -EEXIST;
        /* don't mix zc providers */
        if (unlikely(prev_uarg->ops != &io_ubuf_ops))
                return -EEXIST;

        prev_nd = container_of(prev_uarg, struct io_notif_data, uarg);
        prev_notif = cmd_to_io_kiocb(prev_nd);

        /* make sure all notifications can be finished in the same task_work */
        if (unlikely(notif->ctx != prev_notif->ctx ||
                     notif->tctx != prev_notif->tctx))
                return -EEXIST;

        nd->head = prev_nd->head;
        nd->next = prev_nd->next;
        prev_nd->next = nd;
        net_zcopy_get(&nd->head->uarg);
        return 0;
}

static const struct ubuf_info_ops io_ubuf_ops = {
        .complete = io_tx_ubuf_complete,
        .link_skb = io_link_skb,
};

struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx)
        __must_hold(&ctx->uring_lock)
{
        struct io_kiocb *notif;
        struct io_notif_data *nd;

        if (unlikely(!io_alloc_req(ctx, &notif)))
                return NULL;
        notif->ctx = ctx;
        notif->opcode = IORING_OP_NOP;
        notif->flags = 0;
        notif->file = NULL;
        notif->tctx = current->io_uring;
        io_get_task_refs(1);
        notif->file_node = NULL;
        notif->buf_node = NULL;

        nd = io_notif_to_data(notif);
        nd->zc_report = false;
        nd->account_pages = 0;
        nd->next = NULL;
        nd->head = nd;

        nd->uarg.flags = IO_NOTIF_UBUF_FLAGS;
        nd->uarg.ops = &io_ubuf_ops;
        refcount_set(&nd->uarg.refcnt, 1);
        return notif;
}




























    5 




















    5 




    5 
    5 













































    1 


    1 












    2 
    2 










    1 


    1 












    3 
    3 










    3 

    1 
    3 












    1 
    1 




































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
// SPDX-License-Identifier: GPL-2.0-or-later
/* xfrm4_protocol.c - Generic xfrm protocol multiplexer.
 *
 * Copyright (C) 2013 secunet Security Networks AG
 *
 * Author:
 * Steffen Klassert <steffen.klassert@secunet.com>
 *
 * Based on:
 * net/ipv4/tunnel4.c
 */

#include <linux/init.h>
#include <linux/mutex.h>
#include <linux/skbuff.h>
#include <net/icmp.h>
#include <net/ip.h>
#include <net/protocol.h>
#include <net/xfrm.h>

static struct xfrm4_protocol __rcu *esp4_handlers __read_mostly;
static struct xfrm4_protocol __rcu *ah4_handlers __read_mostly;
static struct xfrm4_protocol __rcu *ipcomp4_handlers __read_mostly;
static DEFINE_MUTEX(xfrm4_protocol_mutex);

static inline struct xfrm4_protocol __rcu **proto_handlers(u8 protocol)
{
        switch (protocol) {
        case IPPROTO_ESP:
                return &esp4_handlers;
        case IPPROTO_AH:
                return &ah4_handlers;
        case IPPROTO_COMP:
                return &ipcomp4_handlers;
        }

        return NULL;
}

#define for_each_protocol_rcu(head, handler)                \
        for (handler = rcu_dereference(head);                \
             handler != NULL;                                \
             handler = rcu_dereference(handler->next))        \

static int xfrm4_rcv_cb(struct sk_buff *skb, u8 protocol, int err)
{
        int ret;
        struct xfrm4_protocol *handler;
        struct xfrm4_protocol __rcu **head = proto_handlers(protocol);

        if (!head)
                return 0;

        for_each_protocol_rcu(*head, handler)
                if ((ret = handler->cb_handler(skb, err)) <= 0)
                        return ret;

        return 0;
}

int xfrm4_rcv_encap(struct sk_buff *skb, int nexthdr, __be32 spi,
                    int encap_type)
{
        int ret;
        struct xfrm4_protocol *handler;
        struct xfrm4_protocol __rcu **head = proto_handlers(nexthdr);

        XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4 = NULL;
        XFRM_SPI_SKB_CB(skb)->family = AF_INET;
        XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct iphdr, daddr);

        if (!head)
                goto out;

        if (!skb_dst(skb)) {
                const struct iphdr *iph = ip_hdr(skb);

                if (ip_route_input_noref(skb, iph->daddr, iph->saddr,
                                         ip4h_dscp(iph), skb->dev))
                        goto drop;
        }

        for_each_protocol_rcu(*head, handler)
                if ((ret = handler->input_handler(skb, nexthdr, spi, encap_type)) != -EINVAL)
                        return ret;

out:
        icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);

drop:
        kfree_skb(skb);
        return 0;
}
EXPORT_SYMBOL(xfrm4_rcv_encap);

static int xfrm4_esp_rcv(struct sk_buff *skb)
{
        int ret;
        struct xfrm4_protocol *handler;

        XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4 = NULL;

        for_each_protocol_rcu(esp4_handlers, handler)
                if ((ret = handler->handler(skb)) != -EINVAL)
                        return ret;

        icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);

        kfree_skb(skb);
        return 0;
}

static int xfrm4_esp_err(struct sk_buff *skb, u32 info)
{
        struct xfrm4_protocol *handler;

        for_each_protocol_rcu(esp4_handlers, handler)
                if (!handler->err_handler(skb, info))
                        return 0;

        return -ENOENT;
}

static int xfrm4_ah_rcv(struct sk_buff *skb)
{
        int ret;
        struct xfrm4_protocol *handler;

        XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4 = NULL;

        for_each_protocol_rcu(ah4_handlers, handler)
                if ((ret = handler->handler(skb)) != -EINVAL)
                        return ret;

        icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);

        kfree_skb(skb);
        return 0;
}

static int xfrm4_ah_err(struct sk_buff *skb, u32 info)
{
        struct xfrm4_protocol *handler;

        for_each_protocol_rcu(ah4_handlers, handler)
                if (!handler->err_handler(skb, info))
                        return 0;

        return -ENOENT;
}

static int xfrm4_ipcomp_rcv(struct sk_buff *skb)
{
        int ret;
        struct xfrm4_protocol *handler;

        XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4 = NULL;

        for_each_protocol_rcu(ipcomp4_handlers, handler)
                if ((ret = handler->handler(skb)) != -EINVAL)
                        return ret;

        icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);

        kfree_skb(skb);
        return 0;
}

static int xfrm4_ipcomp_err(struct sk_buff *skb, u32 info)
{
        struct xfrm4_protocol *handler;

        for_each_protocol_rcu(ipcomp4_handlers, handler)
                if (!handler->err_handler(skb, info))
                        return 0;

        return -ENOENT;
}

static const struct net_protocol esp4_protocol = {
        .handler        =        xfrm4_esp_rcv,
        .err_handler        =        xfrm4_esp_err,
        .no_policy        =        1,
};

static const struct net_protocol ah4_protocol = {
        .handler        =        xfrm4_ah_rcv,
        .err_handler        =        xfrm4_ah_err,
        .no_policy        =        1,
};

static const struct net_protocol ipcomp4_protocol = {
        .handler        =        xfrm4_ipcomp_rcv,
        .err_handler        =        xfrm4_ipcomp_err,
        .no_policy        =        1,
};

static const struct xfrm_input_afinfo xfrm4_input_afinfo = {
        .family                =        AF_INET,
        .callback        =        xfrm4_rcv_cb,
};

static inline const struct net_protocol *netproto(unsigned char protocol)
{
        switch (protocol) {
        case IPPROTO_ESP:
                return &esp4_protocol;
        case IPPROTO_AH:
                return &ah4_protocol;
        case IPPROTO_COMP:
                return &ipcomp4_protocol;
        }

        return NULL;
}

int xfrm4_protocol_register(struct xfrm4_protocol *handler,
                            unsigned char protocol)
{
        struct xfrm4_protocol __rcu **pprev;
        struct xfrm4_protocol *t;
        bool add_netproto = false;
        int ret = -EEXIST;
        int priority = handler->priority;

        if (!proto_handlers(protocol) || !netproto(protocol))
                return -EINVAL;

        mutex_lock(&xfrm4_protocol_mutex);

        if (!rcu_dereference_protected(*proto_handlers(protocol),
                                       lockdep_is_held(&xfrm4_protocol_mutex)))
                add_netproto = true;

        for (pprev = proto_handlers(protocol);
             (t = rcu_dereference_protected(*pprev,
                        lockdep_is_held(&xfrm4_protocol_mutex))) != NULL;
             pprev = &t->next) {
                if (t->priority < priority)
                        break;
                if (t->priority == priority)
                        goto err;
        }

        handler->next = *pprev;
        rcu_assign_pointer(*pprev, handler);

        ret = 0;

err:
        mutex_unlock(&xfrm4_protocol_mutex);

        if (add_netproto) {
                if (inet_add_protocol(netproto(protocol), protocol)) {
                        pr_err("%s: can't add protocol\n", __func__);
                        ret = -EAGAIN;
                }
        }

        return ret;
}
EXPORT_SYMBOL(xfrm4_protocol_register);

int xfrm4_protocol_deregister(struct xfrm4_protocol *handler,
                              unsigned char protocol)
{
        struct xfrm4_protocol __rcu **pprev;
        struct xfrm4_protocol *t;
        int ret = -ENOENT;

        if (!proto_handlers(protocol) || !netproto(protocol))
                return -EINVAL;

        mutex_lock(&xfrm4_protocol_mutex);

        for (pprev = proto_handlers(protocol);
             (t = rcu_dereference_protected(*pprev,
                        lockdep_is_held(&xfrm4_protocol_mutex))) != NULL;
             pprev = &t->next) {
                if (t == handler) {
                        *pprev = handler->next;
                        ret = 0;
                        break;
                }
        }

        if (!rcu_dereference_protected(*proto_handlers(protocol),
                                       lockdep_is_held(&xfrm4_protocol_mutex))) {
                if (inet_del_protocol(netproto(protocol), protocol) < 0) {
                        pr_err("%s: can't remove protocol\n", __func__);
                        ret = -EAGAIN;
                }
        }

        mutex_unlock(&xfrm4_protocol_mutex);

        synchronize_net();

        return ret;
}
EXPORT_SYMBOL(xfrm4_protocol_deregister);

void __init xfrm4_protocol_init(void)
{
        xfrm_input_register_afinfo(&xfrm4_input_afinfo);
}









































    3 











    3 


    3 


    3 


















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
// SPDX-License-Identifier: GPL-2.0-or-later
/* General filesystem local caching manager
 *
 * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#define FSCACHE_DEBUG_LEVEL CACHE
#include <linux/module.h>
#include <linux/init.h>
#include "internal.h"
#define CREATE_TRACE_POINTS
#include <trace/events/fscache.h>

EXPORT_TRACEPOINT_SYMBOL(fscache_access_cache);
EXPORT_TRACEPOINT_SYMBOL(fscache_access_volume);
EXPORT_TRACEPOINT_SYMBOL(fscache_access);

struct workqueue_struct *fscache_wq;
EXPORT_SYMBOL(fscache_wq);

/*
 * Mixing scores (in bits) for (7,20):
 * Input delta: 1-bit      2-bit
 * 1 round:     330.3     9201.6
 * 2 rounds:   1246.4    25475.4
 * 3 rounds:   1907.1    31295.1
 * 4 rounds:   2042.3    31718.6
 * Perfect:    2048      31744
 *            (32*64)   (32*31/2 * 64)
 */
#define HASH_MIX(x, y, a)        \
        (        x ^= (a),        \
        y ^= x,        x = rol32(x, 7),\
        x += y,        y = rol32(y,20),\
        y *= 9                        )

static inline unsigned int fold_hash(unsigned long x, unsigned long y)
{
        /* Use arch-optimized multiply if one exists */
        return __hash_32(y ^ __hash_32(x));
}

/*
 * Generate a hash.  This is derived from full_name_hash(), but we want to be
 * sure it is arch independent and that it doesn't change as bits of the
 * computed hash value might appear on disk.  The caller must guarantee that
 * the source data is a multiple of four bytes in size.
 */
unsigned int fscache_hash(unsigned int salt, const void *data, size_t len)
{
        const __le32 *p = data;
        unsigned int a, x = 0, y = salt, n = len / sizeof(__le32);

        for (; n; n--) {
                a = le32_to_cpu(*p++);
                HASH_MIX(x, y, a);
        }
        return fold_hash(x, y);
}

/*
 * initialise the fs caching module
 */
int __init fscache_init(void)
{
        int ret = -ENOMEM;

        fscache_wq = alloc_workqueue("fscache", WQ_UNBOUND | WQ_FREEZABLE, 0);
        if (!fscache_wq)
                goto error_wq;

        ret = fscache_proc_init();
        if (ret < 0)
                goto error_proc;

        fscache_cookie_jar = kmem_cache_create("fscache_cookie_jar",
                                               sizeof(struct fscache_cookie),
                                               0, 0, NULL);
        if (!fscache_cookie_jar) {
                pr_notice("Failed to allocate a cookie jar\n");
                ret = -ENOMEM;
                goto error_cookie_jar;
        }

        pr_notice("FS-Cache loaded\n");
        return 0;

error_cookie_jar:
        fscache_proc_cleanup();
error_proc:
        destroy_workqueue(fscache_wq);
error_wq:
        return ret;
}

/*
 * clean up on module removal
 */
void __exit fscache_exit(void)
{
        _enter("");

        kmem_cache_destroy(fscache_cookie_jar);
        fscache_proc_cleanup();
        timer_shutdown_sync(&fscache_cookie_lru_timer);
        destroy_workqueue(fscache_wq);
        pr_notice("FS-Cache unloaded\n");
}






































































































































































































































 1013 







    1 





    1 

















   57 
   20 


   41 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 *        inet6 interface/address list definitions
 *        Linux INET6 implementation 
 *
 *        Authors:
 *        Pedro Roque                <roque@di.fc.ul.pt>        
 */

#ifndef _NET_IF_INET6_H
#define _NET_IF_INET6_H

#include <net/snmp.h>
#include <linux/ipv6.h>
#include <linux/refcount.h>

/* inet6_dev.if_flags */

#define IF_RA_OTHERCONF        0x80
#define IF_RA_MANAGED        0x40
#define IF_RA_RCVD        0x20
#define IF_RS_SENT        0x10
#define IF_READY        0x80000000

enum {
        INET6_IFADDR_STATE_PREDAD,
        INET6_IFADDR_STATE_DAD,
        INET6_IFADDR_STATE_POSTDAD,
        INET6_IFADDR_STATE_ERRDAD,
        INET6_IFADDR_STATE_DEAD,
};

struct inet6_ifaddr {
        struct in6_addr                addr;
        __u32                        prefix_len;
        __u32                        rt_priority;

        /* In seconds, relative to tstamp. Expiry is at tstamp + HZ * lft. */
        __u32                        valid_lft;
        __u32                        prefered_lft;
        refcount_t                refcnt;
        spinlock_t                lock;

        int                        state;

        __u32                        flags;
        __u8                        dad_probes;
        __u8                        stable_privacy_retry;

        __u16                        scope;
        __u64                        dad_nonce;

        unsigned long                cstamp;        /* created timestamp */
        unsigned long                tstamp; /* updated timestamp */

        struct delayed_work        dad_work;

        struct inet6_dev        *idev;
        struct fib6_info        *rt;

        struct hlist_node        addr_lst;
        struct list_head        if_list;
        /*
         * Used to safely traverse idev->addr_list in process context
         * if the idev->lock needed to protect idev->addr_list cannot be held.
         * In that case, add the items to this list temporarily and iterate
         * without holding idev->lock.
         * See addrconf_ifdown and dev_forward_change.
         */
        struct list_head        if_list_aux;

        struct list_head        tmp_list;
        struct inet6_ifaddr        *ifpub;
        int                        regen_count;

        bool                        tokenized;

        u8                        ifa_proto;

        struct rcu_head                rcu;
        struct in6_addr                peer_addr;
};

struct ip6_sf_socklist {
        unsigned int                sl_max;
        unsigned int                sl_count;
        struct rcu_head                rcu;
        struct in6_addr                sl_addr[] __counted_by(sl_max);
};

struct ipv6_mc_socklist {
        struct in6_addr                addr;
        int                        ifindex;
        unsigned int                sfmode;                /* MCAST_{INCLUDE,EXCLUDE} */
        struct ipv6_mc_socklist __rcu *next;
        struct ip6_sf_socklist        __rcu *sflist;
        struct rcu_head                rcu;
};

struct ip6_sf_list {
        struct ip6_sf_list __rcu *sf_next;
        struct in6_addr                sf_addr;
        unsigned long                sf_count[2];        /* include/exclude counts */
        unsigned char                sf_gsresp;        /* include in g & s response? */
        unsigned char                sf_oldin;        /* change state */
        unsigned char                sf_crcount;        /* retrans. left to send */
        struct rcu_head                rcu;
};

#define MAF_TIMER_RUNNING        0x01
#define MAF_LAST_REPORTER        0x02
#define MAF_LOADED                0x04
#define MAF_NOREPORT                0x08
#define MAF_GSQUERY                0x10

struct ifmcaddr6 {
        struct in6_addr                mca_addr;
        struct inet6_dev        *idev;
        struct ifmcaddr6        __rcu *next;
        struct ip6_sf_list        __rcu *mca_sources;
        struct ip6_sf_list        __rcu *mca_tomb;
        unsigned int                mca_sfmode;
        unsigned char                mca_crcount;
        unsigned long                mca_sfcount[2];
        struct delayed_work        mca_work;
        unsigned int                mca_flags;
        int                        mca_users;
        refcount_t                mca_refcnt;
        unsigned long                mca_cstamp;
        unsigned long                mca_tstamp;
        struct rcu_head                rcu;
};

/* Anycast stuff */

struct ipv6_ac_socklist {
        struct in6_addr                acl_addr;
        int                        acl_ifindex;
        struct ipv6_ac_socklist *acl_next;
};

struct ifacaddr6 {
        struct in6_addr                aca_addr;
        struct fib6_info        *aca_rt;
        struct ifacaddr6 __rcu        *aca_next;
        struct hlist_node        aca_addr_lst;
        int                        aca_users;
        refcount_t                aca_refcnt;
        unsigned long                aca_cstamp;
        unsigned long                aca_tstamp;
        struct rcu_head                rcu;
};

#define        IFA_HOST        IPV6_ADDR_LOOPBACK
#define        IFA_LINK        IPV6_ADDR_LINKLOCAL
#define        IFA_SITE        IPV6_ADDR_SITELOCAL

struct ipv6_devstat {
        struct proc_dir_entry        *proc_dir_entry;
        DEFINE_SNMP_STAT(struct ipstats_mib, ipv6);
        DEFINE_SNMP_STAT_ATOMIC(struct icmpv6_mib_device, icmpv6dev);
        DEFINE_SNMP_STAT_ATOMIC(struct icmpv6msg_mib_device, icmpv6msgdev);
};

struct inet6_dev {
        struct net_device        *dev;
        netdevice_tracker        dev_tracker;

        struct list_head        addr_list;

        struct ifmcaddr6        __rcu *mc_list;
        struct ifmcaddr6        __rcu *mc_tomb;

        unsigned char                mc_qrv;                /* Query Robustness Variable */
        unsigned char                mc_gq_running;
        unsigned char                mc_ifc_count;
        unsigned char                mc_dad_count;

        unsigned long                mc_v1_seen;        /* Max time we stay in MLDv1 mode */
        unsigned long                mc_qi;                /* Query Interval */
        unsigned long                mc_qri;                /* Query Response Interval */
        unsigned long                mc_maxdelay;

        struct delayed_work        mc_gq_work;        /* general query work */
        struct delayed_work        mc_ifc_work;        /* interface change work */
        struct delayed_work        mc_dad_work;        /* dad complete mc work */
        struct delayed_work        mc_query_work;        /* mld query work */
        struct delayed_work        mc_report_work;        /* mld report work */

        struct sk_buff_head        mc_query_queue;                /* mld query queue */
        struct sk_buff_head        mc_report_queue;        /* mld report queue */

        spinlock_t                mc_query_lock;        /* mld query queue lock */
        spinlock_t                mc_report_lock;        /* mld query report lock */
        struct mutex                mc_lock;        /* mld global lock */

        struct ifacaddr6 __rcu        *ac_list;
        rwlock_t                lock;
        refcount_t                refcnt;
        __u32                        if_flags;
        int                        dead;

        u32                        desync_factor;
        struct list_head        tempaddr_list;

        struct in6_addr                token;

        struct neigh_parms        *nd_parms;
        struct ipv6_devconf        cnf;
        struct ipv6_devstat        stats;

        struct timer_list        rs_timer;
        __s32                        rs_interval;        /* in jiffies */
        __u8                        rs_probes;

        unsigned long                tstamp; /* ipv6InterfaceTable update timestamp */
        struct rcu_head                rcu;

        unsigned int                ra_mtu;
};

static inline void ipv6_eth_mc_map(const struct in6_addr *addr, char *buf)
{
        /*
         *        +-------+-------+-------+-------+-------+-------+
         *      |   33  |   33  | DST13 | DST14 | DST15 | DST16 |
         *      +-------+-------+-------+-------+-------+-------+
         */

        buf[0]= 0x33;
        buf[1]= 0x33;

        memcpy(buf + 2, &addr->s6_addr32[3], sizeof(__u32));
}

static inline void ipv6_arcnet_mc_map(const struct in6_addr *addr, char *buf)
{
        buf[0] = 0x00;
}

static inline void ipv6_ib_mc_map(const struct in6_addr *addr,
                                  const unsigned char *broadcast, char *buf)
{
        unsigned char scope = broadcast[5] & 0xF;

        buf[0]  = 0;                /* Reserved */
        buf[1]  = 0xff;                /* Multicast QPN */
        buf[2]  = 0xff;
        buf[3]  = 0xff;
        buf[4]  = 0xff;
        buf[5]  = 0x10 | scope;        /* scope from broadcast address */
        buf[6]  = 0x60;                /* IPv6 signature */
        buf[7]  = 0x1b;
        buf[8]  = broadcast[8];        /* P_Key */
        buf[9]  = broadcast[9];
        memcpy(buf + 10, addr->s6_addr + 6, 10);
}

static inline int ipv6_ipgre_mc_map(const struct in6_addr *addr,
                                    const unsigned char *broadcast, char *buf)
{
        if ((broadcast[0] | broadcast[1] | broadcast[2] | broadcast[3]) != 0) {
                memcpy(buf, broadcast, 4);
        } else {
                /* v4mapped? */
                if ((addr->s6_addr32[0] | addr->s6_addr32[1] |
                     (addr->s6_addr32[2] ^ htonl(0x0000ffff))) != 0)
                        return -EINVAL;
                memcpy(buf, &addr->s6_addr32[3], 4);
        }
        return 0;
}

#endif


























































   11 






































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2025 Christian Brauner <brauner@kernel.org> */
#ifndef _LINUX_NSTREE_H
#define _LINUX_NSTREE_H

#include <linux/ns/nstree_types.h>
#include <linux/nsproxy.h>
#include <linux/rbtree.h>
#include <linux/seqlock.h>
#include <linux/rculist.h>
#include <linux/cookie.h>
#include <uapi/linux/nsfs.h>

struct ns_common;

extern struct ns_tree_root cgroup_ns_tree;
extern struct ns_tree_root ipc_ns_tree;
extern struct ns_tree_root mnt_ns_tree;
extern struct ns_tree_root net_ns_tree;
extern struct ns_tree_root pid_ns_tree;
extern struct ns_tree_root time_ns_tree;
extern struct ns_tree_root user_ns_tree;
extern struct ns_tree_root uts_ns_tree;

void ns_tree_node_init(struct ns_tree_node *node);
void ns_tree_root_init(struct ns_tree_root *root);
bool ns_tree_node_empty(const struct ns_tree_node *node);
struct rb_node *ns_tree_node_add(struct ns_tree_node *node,
                                  struct ns_tree_root *root,
                                  int (*cmp)(struct rb_node *, const struct rb_node *));
void ns_tree_node_del(struct ns_tree_node *node, struct ns_tree_root *root);

#define to_ns_tree(__ns)                                        \
        _Generic((__ns),                                        \
                struct cgroup_namespace *: &(cgroup_ns_tree),        \
                struct ipc_namespace *:    &(ipc_ns_tree),        \
                struct net *:              &(net_ns_tree),        \
                struct pid_namespace *:    &(pid_ns_tree),        \
                struct mnt_namespace *:    &(mnt_ns_tree),        \
                struct time_namespace *:   &(time_ns_tree),        \
                struct user_namespace *:   &(user_ns_tree),        \
                struct uts_namespace *:    &(uts_ns_tree))

#define ns_tree_gen_id(__ns)                 \
        __ns_tree_gen_id(to_ns_common(__ns), \
                         (((__ns) == ns_init_ns(__ns)) ? ns_init_id(__ns) : 0))

u64 __ns_tree_gen_id(struct ns_common *ns, u64 id);
void __ns_tree_add_raw(struct ns_common *ns, struct ns_tree_root *ns_tree);
void __ns_tree_remove(struct ns_common *ns, struct ns_tree_root *ns_tree);
struct ns_common *ns_tree_lookup_rcu(u64 ns_id, int ns_type);
struct ns_common *__ns_tree_adjoined_rcu(struct ns_common *ns,
                                         struct ns_tree_root *ns_tree,
                                         bool previous);

static inline void __ns_tree_add(struct ns_common *ns, struct ns_tree_root *ns_tree, u64 id)
{
        __ns_tree_gen_id(ns, id);
        __ns_tree_add_raw(ns, ns_tree);
}

/**
 * ns_tree_add_raw - Add a namespace to a namespace
 * @__ns: Namespace to add
 *
 * This function adds a namespace to the appropriate namespace tree
 * without assigning a id.
 */
#define ns_tree_add_raw(__ns) __ns_tree_add_raw(to_ns_common(__ns), to_ns_tree(__ns))

/**
 * ns_tree_add - Add a namespace to a namespace tree
 * @__ns: Namespace to add
 *
 * This function assigns a new id to the namespace and adds it to the
 * appropriate namespace tree and list.
 */
#define ns_tree_add(__ns)                                   \
        __ns_tree_add(to_ns_common(__ns), to_ns_tree(__ns), \
                      (((__ns) == ns_init_ns(__ns)) ? ns_init_id(__ns) : 0))

/**
 * ns_tree_remove - Remove a namespace from a namespace tree
 * @__ns: Namespace to remove
 *
 * This function removes a namespace from the appropriate namespace
 * tree and list.
 */
#define ns_tree_remove(__ns)  __ns_tree_remove(to_ns_common(__ns), to_ns_tree(__ns))

#define ns_tree_adjoined_rcu(__ns, __previous) \
        __ns_tree_adjoined_rcu(to_ns_common(__ns), to_ns_tree(__ns), __previous)

#define ns_tree_active(__ns) (!RB_EMPTY_NODE(&to_ns_common(__ns)->ns_tree_node.ns_node))

#endif /* _LINUX_NSTREE_H */










































































































































































































































































































































































































































































































































































































  268 



























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_TTY_DRIVER_H
#define _LINUX_TTY_DRIVER_H

#include <linux/export.h>
#include <linux/fs.h>
#include <linux/kref.h>
#include <linux/list.h>
#include <linux/cdev.h>
#include <linux/uaccess.h>
#include <linux/termios.h>
#include <linux/seq_file.h>

struct tty_struct;
struct tty_driver;
struct serial_icounter_struct;
struct serial_struct;

/**
 * enum tty_driver_flag -- TTY Driver Flags
 *
 * These are flags passed to tty_alloc_driver().
 *
 * @TTY_DRIVER_INSTALLED:
 *        Whether this driver was succesfully installed. This is a tty internal
 *        flag. Do not touch.
 *
 * @TTY_DRIVER_RESET_TERMIOS:
 *        Requests the tty layer to reset the termios setting when the last
 *        process has closed the device. Used for PTYs, in particular.
 *
 * @TTY_DRIVER_REAL_RAW:
 *        Indicates that the driver will guarantee not to set any special
 *        character handling flags if this is set for the tty:
 *
 *        ``(IGNBRK || (!BRKINT && !PARMRK)) && (IGNPAR || !INPCK)``
 *
 *        That is, if there is no reason for the driver to
 *        send notifications of parity and break characters up to the line
 *        driver, it won't do so.  This allows the line driver to optimize for
 *        this case if this flag is set.  (Note that there is also a promise, if
 *        the above case is true, not to signal overruns, either.)
 *
 * @TTY_DRIVER_DYNAMIC_DEV:
 *        The individual tty devices need to be registered with a call to
 *        tty_register_device() when the device is found in the system and
 *        unregistered with a call to tty_unregister_device() so the devices will
 *        be show up properly in sysfs.  If not set, all &tty_driver.num entries
 *        will be created by the tty core in sysfs when tty_register_driver() is
 *        called.  This is to be used by drivers that have tty devices that can
 *        appear and disappear while the main tty driver is registered with the
 *        tty core.
 *
 * @TTY_DRIVER_DEVPTS_MEM:
 *        Don't use the standard arrays (&tty_driver.ttys and
 *        &tty_driver.termios), instead use dynamic memory keyed through the
 *        devpts filesystem. This is only applicable to the PTY driver.
 *
 * @TTY_DRIVER_HARDWARE_BREAK:
 *        Hardware handles break signals. Pass the requested timeout to the
 *        &tty_operations.break_ctl instead of using a simple on/off interface.
 *
 * @TTY_DRIVER_DYNAMIC_ALLOC:
 *        Do not allocate structures which are needed per line for this driver
 *        (&tty_driver.ports) as it would waste memory. The driver will take
 *        care. This is only applicable to the PTY driver.
 *
 * @TTY_DRIVER_UNNUMBERED_NODE:
 *        Do not create numbered ``/dev`` nodes. For example, create
 *        ``/dev/ttyprintk`` and not ``/dev/ttyprintk0``. Applicable only when a
 *        driver for a single tty device is being allocated.
 *
 * @TTY_DRIVER_NO_WORKQUEUE:
 *        Do not create workqueue when tty_register_driver(). Whenever set, flip
 *        buffer workqueue can be set by tty_port_link_wq() for every port.
 */
enum tty_driver_flag {
        TTY_DRIVER_INSTALLED                = BIT(0),
        TTY_DRIVER_RESET_TERMIOS        = BIT(1),
        TTY_DRIVER_REAL_RAW                = BIT(2),
        TTY_DRIVER_DYNAMIC_DEV                = BIT(3),
        TTY_DRIVER_DEVPTS_MEM                = BIT(4),
        TTY_DRIVER_HARDWARE_BREAK        = BIT(5),
        TTY_DRIVER_DYNAMIC_ALLOC        = BIT(6),
        TTY_DRIVER_UNNUMBERED_NODE        = BIT(7),
        TTY_DRIVER_NO_WORKQUEUE                = BIT(8),
};

enum tty_driver_type {
        TTY_DRIVER_TYPE_SYSTEM,
        TTY_DRIVER_TYPE_CONSOLE,
        TTY_DRIVER_TYPE_SERIAL,
        TTY_DRIVER_TYPE_PTY,
        TTY_DRIVER_TYPE_SCC,
        TTY_DRIVER_TYPE_SYSCONS,
};

enum tty_driver_subtype {
        SYSTEM_TYPE_TTY = 1,
        SYSTEM_TYPE_CONSOLE,
        SYSTEM_TYPE_SYSCONS,
        SYSTEM_TYPE_SYSPTMX,

        PTY_TYPE_MASTER = 1,
        PTY_TYPE_SLAVE,

        SERIAL_TYPE_NORMAL = 1,
};

/**
 * struct tty_operations -- interface between driver and tty
 *
 * @lookup: ``struct tty_struct *()(struct tty_driver *self, struct file *,
 *                                    int idx)``
 *
 *        Return the tty device corresponding to @idx, %NULL if there is not
 *        one currently in use and an %ERR_PTR value on error. Called under
 *        %tty_mutex (for now!)
 *
 *        Optional method. Default behaviour is to use the @self->ttys array.
 *
 * @install: ``int ()(struct tty_driver *self, struct tty_struct *tty)``
 *
 *        Install a new @tty into the @self's internal tables. Used in
 *        conjunction with @lookup and @remove methods.
 *
 *        Optional method. Default behaviour is to use the @self->ttys array.
 *
 * @remove: ``void ()(struct tty_driver *self, struct tty_struct *tty)``
 *
 *        Remove a closed @tty from the @self's internal tables. Used in
 *        conjunction with @lookup and @remove methods.
 *
 *        Optional method. Default behaviour is to use the @self->ttys array.
 *
 * @open: ``int ()(struct tty_struct *tty, struct file *)``
 *
 *        This routine is called when a particular @tty device is opened. This
 *        routine is mandatory; if this routine is not filled in, the attempted
 *        open will fail with %ENODEV.
 *
 *        Required method. Called with tty lock held. May sleep.
 *
 * @close: ``void ()(struct tty_struct *tty, struct file *)``
 *
 *        This routine is called when a particular @tty device is closed. At the
 *        point of return from this call the driver must make no further ldisc
 *        calls of any kind.
 *
 *        Remark: called even if the corresponding @open() failed.
 *
 *        Required method. Called with tty lock held. May sleep.
 *
 * @shutdown: ``void ()(struct tty_struct *tty)``
 *
 *        This routine is called under the tty lock when a particular @tty device
 *        is closed for the last time. It executes before the @tty resources
 *        are freed so may execute while another function holds a @tty kref.
 *
 * @cleanup: ``void ()(struct tty_struct *tty)``
 *
 *        This routine is called asynchronously when a particular @tty device
 *        is closed for the last time freeing up the resources. This is
 *        actually the second part of shutdown for routines that might sleep.
 *
 * @write: ``ssize_t ()(struct tty_struct *tty, const u8 *buf, size_t count)``
 *
 *        This routine is called by the kernel to write a series (@count) of
 *        characters (@buf) to the @tty device. The characters may come from
 *        user space or kernel space.  This routine will return the
 *        number of characters actually accepted for writing.
 *
 *        May occur in parallel in special cases. Because this includes panic
 *        paths drivers generally shouldn't try and do clever locking here.
 *
 *        Optional: Required for writable devices. May not sleep.
 *
 * @put_char: ``int ()(struct tty_struct *tty, u8 ch)``
 *
 *        This routine is called by the kernel to write a single character @ch to
 *        the @tty device. If the kernel uses this routine, it must call the
 *        @flush_chars() routine (if defined) when it is done stuffing characters
 *        into the driver. If there is no room in the queue, the character is
 *        ignored.
 *
 *        Optional: Kernel will use the @write method if not provided. Do not
 *        call this function directly, call tty_put_char().
 *
 * @flush_chars: ``void ()(struct tty_struct *tty)``
 *
 *        This routine is called by the kernel after it has written a
 *        series of characters to the tty device using @put_char().
 *
 *        Optional. Do not call this function directly, call
 *        tty_driver_flush_chars().
 *
 * @write_room: ``unsigned int ()(struct tty_struct *tty)``
 *
 *        This routine returns the numbers of characters the @tty driver
 *        will accept for queuing to be written.  This number is subject
 *        to change as output buffers get emptied, or if the output flow
 *        control is acted.
 *
 *        The ldisc is responsible for being intelligent about multi-threading of
 *        write_room/write calls
 *
 *        Required if @write method is provided else not needed. Do not call this
 *        function directly, call tty_write_room()
 *
 * @chars_in_buffer: ``unsigned int ()(struct tty_struct *tty)``
 *
 *        This routine returns the number of characters in the device private
 *        output queue. Used in tty_wait_until_sent() and for poll()
 *        implementation.
 *
 *        Optional: if not provided, it is assumed there is no queue on the
 *        device. Do not call this function directly, call tty_chars_in_buffer().
 *
 * @ioctl: ``int ()(struct tty_struct *tty, unsigned int cmd,
 *                    unsigned long arg)``
 *
 *        This routine allows the @tty driver to implement device-specific
 *        ioctls. If the ioctl number passed in @cmd is not recognized by the
 *        driver, it should return %ENOIOCTLCMD.
 *
 *        Optional.
 *
 * @compat_ioctl: ``long ()(struct tty_struct *tty, unsigned int cmd,
 *                          unsigned long arg)``
 *
 *        Implement ioctl processing for 32 bit process on 64 bit system.
 *
 *        Optional.
 *
 * @set_termios: ``void ()(struct tty_struct *tty, const struct ktermios *old)``
 *
 *        This routine allows the @tty driver to be notified when device's
 *        termios settings have changed. New settings are in @tty->termios.
 *        Previous settings are passed in the @old argument.
 *
 *        The API is defined such that the driver should return the actual modes
 *        selected. This means that the driver is responsible for modifying any
 *        bits in @tty->termios it cannot fulfill to indicate the actual modes
 *        being used.
 *
 *        Optional. Called under the @tty->termios_rwsem. May sleep.
 *
 * @ldisc_ok: ``int ()(struct tty_struct *tty, int ldisc)``
 *
 *        This routine allows the @tty driver to decide if it can deal
 *        with a particular @ldisc.
 *
 *        Optional. Called under the @tty->ldisc_sem and @tty->termios_rwsem.
 *
 * @set_ldisc: ``void ()(struct tty_struct *tty)``
 *
 *        This routine allows the @tty driver to be notified when the device's
 *        line discipline is being changed. At the point this is done the
 *        discipline is not yet usable.
 *
 *        Optional. Called under the @tty->ldisc_sem and @tty->termios_rwsem.
 *
 * @throttle: ``void ()(struct tty_struct *tty)``
 *
 *        This routine notifies the @tty driver that input buffers for the line
 *        discipline are close to full, and it should somehow signal that no more
 *        characters should be sent to the @tty.
 *
 *        Serialization including with @unthrottle() is the job of the ldisc
 *        layer.
 *
 *        Optional: Always invoke via tty_throttle_safe(). Called under the
 *        @tty->termios_rwsem.
 *
 * @unthrottle: ``void ()(struct tty_struct *tty)``
 *
 *        This routine notifies the @tty driver that it should signal that
 *        characters can now be sent to the @tty without fear of overrunning the
 *        input buffers of the line disciplines.
 *
 *        Optional. Always invoke via tty_unthrottle(). Called under the
 *        @tty->termios_rwsem.
 *
 * @stop: ``void ()(struct tty_struct *tty)``
 *
 *        This routine notifies the @tty driver that it should stop outputting
 *        characters to the tty device.
 *
 *        Called with @tty->flow.lock held. Serialized with @start() method.
 *
 *        Optional. Always invoke via stop_tty().
 *
 * @start: ``void ()(struct tty_struct *tty)``
 *
 *        This routine notifies the @tty driver that it resumed sending
 *        characters to the @tty device.
 *
 *        Called with @tty->flow.lock held. Serialized with stop() method.
 *
 *        Optional. Always invoke via start_tty().
 *
 * @hangup: ``void ()(struct tty_struct *tty)``
 *
 *        This routine notifies the @tty driver that it should hang up the @tty
 *        device.
 *
 *        Optional. Called with tty lock held.
 *
 * @break_ctl: ``int ()(struct tty_struct *tty, int state)``
 *
 *        This optional routine requests the @tty driver to turn on or off BREAK
 *        status on the RS-232 port. If @state is -1, then the BREAK status
 *        should be turned on; if @state is 0, then BREAK should be turned off.
 *
 *        If this routine is implemented, the high-level tty driver will handle
 *        the following ioctls: %TCSBRK, %TCSBRKP, %TIOCSBRK, %TIOCCBRK.
 *
 *        If the driver sets %TTY_DRIVER_HARDWARE_BREAK in tty_alloc_driver(),
 *        then the interface will also be called with actual times and the
 *        hardware is expected to do the delay work itself. 0 and -1 are still
 *        used for on/off.
 *
 *        Optional: Required for %TCSBRK/%BRKP/etc. handling. May sleep.
 *
 * @flush_buffer: ``void ()(struct tty_struct *tty)``
 *
 *        This routine discards device private output buffer. Invoked on close,
 *        hangup, to implement %TCOFLUSH ioctl and similar.
 *
 *        Optional: if not provided, it is assumed there is no queue on the
 *        device. Do not call this function directly, call
 *        tty_driver_flush_buffer().
 *
 * @wait_until_sent: ``void ()(struct tty_struct *tty, int timeout)``
 *
 *        This routine waits until the device has written out all of the
 *        characters in its transmitter FIFO. Or until @timeout (in jiffies) is
 *        reached.
 *
 *        Optional: If not provided, the device is assumed to have no FIFO.
 *        Usually correct to invoke via tty_wait_until_sent(). May sleep.
 *
 * @send_xchar: ``void ()(struct tty_struct *tty, u8 ch)``
 *
 *        This routine is used to send a high-priority XON/XOFF character (@ch)
 *        to the @tty device.
 *
 *        Optional: If not provided, then the @write method is called under
 *        the @tty->atomic_write_lock to keep it serialized with the ldisc.
 *
 * @tiocmget: ``int ()(struct tty_struct *tty)``
 *
 *        This routine is used to obtain the modem status bits from the @tty
 *        driver.
 *
 *        Optional: If not provided, then %ENOTTY is returned from the %TIOCMGET
 *        ioctl. Do not call this function directly, call tty_tiocmget().
 *
 * @tiocmset: ``int ()(struct tty_struct *tty,
 *                       unsigned int set, unsigned int clear)``
 *
 *        This routine is used to set the modem status bits to the @tty driver.
 *        First, @clear bits should be cleared, then @set bits set.
 *
 *        Optional: If not provided, then %ENOTTY is returned from the %TIOCMSET
 *        ioctl. Do not call this function directly, call tty_tiocmset().
 *
 * @resize: ``int ()(struct tty_struct *tty, struct winsize *ws)``
 *
 *        Called when a termios request is issued which changes the requested
 *        terminal geometry to @ws.
 *
 *        Optional: the default action is to update the termios structure
 *        without error. This is usually the correct behaviour. Drivers should
 *        not force errors here if they are not resizable objects (e.g. a serial
 *        line). See tty_do_resize() if you need to wrap the standard method
 *        in your own logic -- the usual case.
 *
 * @get_icount: ``int ()(struct tty_struct *tty,
 *                         struct serial_icounter *icount)``
 *
 *        Called when the @tty device receives a %TIOCGICOUNT ioctl. Passed a
 *        kernel structure @icount to complete.
 *
 *        Optional: called only if provided, otherwise %ENOTTY will be returned.
 *
 * @get_serial: ``int ()(struct tty_struct *tty, struct serial_struct *p)``
 *
 *        Called when the @tty device receives a %TIOCGSERIAL ioctl. Passed a
 *        kernel structure @p (&struct serial_struct) to complete.
 *
 *        Optional: called only if provided, otherwise %ENOTTY will be returned.
 *        Do not call this function directly, call tty_tiocgserial().
 *
 * @set_serial: ``int ()(struct tty_struct *tty, struct serial_struct *p)``
 *
 *        Called when the @tty device receives a %TIOCSSERIAL ioctl. Passed a
 *        kernel structure @p (&struct serial_struct) to set the values from.
 *
 *        Optional: called only if provided, otherwise %ENOTTY will be returned.
 *        Do not call this function directly, call tty_tiocsserial().
 *
 * @show_fdinfo: ``void ()(struct tty_struct *tty, struct seq_file *m)``
 *
 *        Called when the @tty device file descriptor receives a fdinfo request
 *        from VFS (to show in /proc/<pid>/fdinfo/). @m should be filled with
 *        information.
 *
 *        Optional: called only if provided, otherwise nothing is written to @m.
 *        Do not call this function directly, call tty_show_fdinfo().
 *
 * @poll_init: ``int ()(struct tty_driver *driver, int line, char *options)``
 *
 *        kgdboc support (Documentation/process/debugging/kgdb.rst). This routine is
 *        called to initialize the HW for later use by calling @poll_get_char or
 *        @poll_put_char.
 *
 *        Optional: called only if provided, otherwise skipped as a non-polling
 *        driver.
 *
 * @poll_get_char: ``int ()(struct tty_driver *driver, int line)``
 *
 *        kgdboc support (see @poll_init). @driver should read a character from a
 *        tty identified by @line and return it.
 *
 *        Optional: called only if @poll_init provided.
 *
 * @poll_put_char: ``void ()(struct tty_driver *driver, int line, char ch)``
 *
 *        kgdboc support (see @poll_init). @driver should write character @ch to
 *        a tty identified by @line.
 *
 *        Optional: called only if @poll_init provided.
 *
 * @proc_show: ``int ()(struct seq_file *m, void *driver)``
 *
 *        Driver @driver (cast to &struct tty_driver) can show additional info in
 *        /proc/tty/driver/<driver_name>. It is enough to fill in the information
 *        into @m.
 *
 *        Optional: called only if provided, otherwise no /proc entry created.
 *
 * This structure defines the interface between the low-level tty driver and
 * the tty routines. These routines can be defined. Unless noted otherwise,
 * they are optional, and can be filled in with a %NULL pointer.
 */
struct tty_operations {
        struct tty_struct * (*lookup)(struct tty_driver *driver,
                        struct file *filp, int idx);
        int  (*install)(struct tty_driver *driver, struct tty_struct *tty);
        void (*remove)(struct tty_driver *driver, struct tty_struct *tty);
        int  (*open)(struct tty_struct * tty, struct file * filp);
        void (*close)(struct tty_struct * tty, struct file * filp);
        void (*shutdown)(struct tty_struct *tty);
        void (*cleanup)(struct tty_struct *tty);
        ssize_t (*write)(struct tty_struct *tty, const u8 *buf, size_t count);
        int  (*put_char)(struct tty_struct *tty, u8 ch);
        void (*flush_chars)(struct tty_struct *tty);
        unsigned int (*write_room)(struct tty_struct *tty);
        unsigned int (*chars_in_buffer)(struct tty_struct *tty);
        int  (*ioctl)(struct tty_struct *tty,
                    unsigned int cmd, unsigned long arg);
        long (*compat_ioctl)(struct tty_struct *tty,
                             unsigned int cmd, unsigned long arg);
        void (*set_termios)(struct tty_struct *tty, const struct ktermios *old);
        void (*throttle)(struct tty_struct * tty);
        void (*unthrottle)(struct tty_struct * tty);
        void (*stop)(struct tty_struct *tty);
        void (*start)(struct tty_struct *tty);
        void (*hangup)(struct tty_struct *tty);
        int (*break_ctl)(struct tty_struct *tty, int state);
        void (*flush_buffer)(struct tty_struct *tty);
        int (*ldisc_ok)(struct tty_struct *tty, int ldisc);
        void (*set_ldisc)(struct tty_struct *tty);
        void (*wait_until_sent)(struct tty_struct *tty, int timeout);
        void (*send_xchar)(struct tty_struct *tty, u8 ch);
        int (*tiocmget)(struct tty_struct *tty);
        int (*tiocmset)(struct tty_struct *tty,
                        unsigned int set, unsigned int clear);
        int (*resize)(struct tty_struct *tty, struct winsize *ws);
        int (*get_icount)(struct tty_struct *tty,
                                struct serial_icounter_struct *icount);
        int  (*get_serial)(struct tty_struct *tty, struct serial_struct *p);
        int  (*set_serial)(struct tty_struct *tty, struct serial_struct *p);
        void (*show_fdinfo)(struct tty_struct *tty, struct seq_file *m);
#ifdef CONFIG_CONSOLE_POLL
        int (*poll_init)(struct tty_driver *driver, int line, char *options);
        int (*poll_get_char)(struct tty_driver *driver, int line);
        void (*poll_put_char)(struct tty_driver *driver, int line, char ch);
#endif
        int (*proc_show)(struct seq_file *m, void *driver);
} __randomize_layout;

/**
 * struct tty_driver -- driver for TTY devices
 *
 * @kref: reference counting. Reaching zero frees all the internals and the
 *          driver.
 * @cdevs: allocated/registered character /dev devices
 * @owner: modules owning this driver. Used drivers cannot be rmmod'ed.
 *           Automatically set by tty_alloc_driver().
 * @driver_name: name of the driver used in /proc/tty
 * @name: used for constructing /dev node name
 * @name_base: used as a number base for constructing /dev node name
 * @major: major /dev device number (zero for autoassignment)
 * @minor_start: the first minor /dev device number
 * @num: number of devices allocated
 * @type: type of tty driver (enum tty_driver_type)
 * @subtype: subtype of tty driver (enum tty_driver_subtype)
 * @init_termios: termios to set to each tty initially (e.g. %tty_std_termios)
 * @flags: tty driver flags (%TTY_DRIVER_)
 * @proc_entry: proc fs entry, used internally
 * @other: driver of the linked tty; only used for the PTY driver
 * @flip_wq: workqueue to queue flip buffer work on
 * @ttys: array of active &struct tty_struct, set by tty_standard_install()
 * @ports: array of &struct tty_port; can be set during initialization by
 *           tty_port_link_device() and similar
 * @termios: storage for termios at each TTY close for the next open
 * @driver_state: pointer to driver's arbitrary data
 * @ops: driver hooks for TTYs. Set them using tty_set_operations(). Use &struct
 *         tty_port helpers in them as much as possible.
 * @tty_drivers: used internally to link tty_drivers together
 *
 * The usual handling of &struct tty_driver is to allocate it by
 * tty_alloc_driver(), set up all the necessary members, and register it by
 * tty_register_driver(). At last, the driver is torn down by calling
 * tty_unregister_driver() followed by tty_driver_kref_put().
 *
 * The fields required to be set before calling tty_register_driver() include
 * @driver_name, @name, @type, @subtype, @init_termios, and @ops.
 */
struct tty_driver {
        struct kref kref;
        struct cdev **cdevs;
        struct module        *owner;
        const char        *driver_name;
        const char        *name;
        int        name_base;
        int        major;
        int        minor_start;
        unsigned int        num;
        enum tty_driver_type type;
        enum tty_driver_subtype subtype;
        struct ktermios init_termios;
        unsigned long        flags;
        struct proc_dir_entry *proc_entry;
        struct tty_driver *other;
        struct workqueue_struct *flip_wq;

        /*
         * Pointer to the tty data structures
         */
        struct tty_struct **ttys;
        struct tty_port **ports;
        struct ktermios **termios;
        void *driver_state;

        /*
         * Driver methods
         */

        const struct tty_operations *ops;
        struct list_head tty_drivers;
} __randomize_layout;

extern struct list_head tty_drivers;

struct tty_driver *__tty_alloc_driver(unsigned int lines, struct module *owner,
                unsigned long flags);
struct tty_driver *tty_find_polling_driver(char *name, int *line);

void tty_driver_kref_put(struct tty_driver *driver);

/**
 * tty_alloc_driver - allocate tty driver
 * @lines: count of lines this driver can handle at most
 * @flags: some of enum tty_driver_flag, will be set in driver->flags
 *
 * Returns: struct tty_driver or a PTR-encoded error (use IS_ERR() and friends).
 */
#define tty_alloc_driver(lines, flags) \
                __tty_alloc_driver(lines, THIS_MODULE, flags)

static inline struct tty_driver *tty_driver_kref_get(struct tty_driver *d)
{
        kref_get(&d->kref);
        return d;
}

static inline void tty_set_operations(struct tty_driver *driver,
                const struct tty_operations *op)
{
        driver->ops = op;
}

int tty_register_driver(struct tty_driver *driver);
void tty_unregister_driver(struct tty_driver *driver);
struct device *tty_register_device(struct tty_driver *driver, unsigned index,
                struct device *dev);
struct device *tty_register_device_attr(struct tty_driver *driver,
                unsigned index, struct device *device, void *drvdata,
                const struct attribute_group **attr_grp);
void tty_unregister_device(struct tty_driver *driver, unsigned index);

#ifdef CONFIG_PROC_FS
void proc_tty_register_driver(struct tty_driver *);
void proc_tty_unregister_driver(struct tty_driver *);
#else
static inline void proc_tty_register_driver(struct tty_driver *d) {}
static inline void proc_tty_unregister_driver(struct tty_driver *d) {}
#endif

#endif /* #ifdef _LINUX_TTY_DRIVER_H */
























































































 13030 




































































 3120 
 3120 


    2 



    2 





























   12 





   12 
   12 
   12 



   12 





    7 







    7 
    7 



















    7 
    7 







   61 












































 7780 

 7780 


 7780 

 7780 



 7780 


    1 


 7780 




 7940 






 7780 
 7780 
 7940 











 7940 

 12627 





 8519 
 8519 




   24 

   23 
   20 



   24 
   24 





  516 
  395 
  516 
  514 




 8394 
    1 

    1 

    1 

 8394 
 8393 
 8394 



 12628 
 12628 


 12628 



    1 






    1 
    1 
    1 











  452 




   29 



   29 


   29 




   29 







  509 










   36 





 7108 

   35 
 7108 


















 8522 
 8573 





 8672 







 8539 


 8539 








 7494 












 8522 
 8523 









   44 








   44 

   44 
   35 
   35 

   44 







 7494 





 7494 



 7494 

 7494 

 7494 






    4 

    4 











 7494 







  408 
  408 



  408 
  408 
    7 
    7 

  408 
  316 
  408 














  630 





  630 










  630 












  575 




  573 

  572 




    3 
    3 




  569 
  569 


  572 


  572 

  572 


  569 



  569 



  575 






    1 



















  410 





  408 








  320 




  408 


  370 
    5 
    5 

  408 
  316 
  319 

  408 




  363 

  363 








  408 
   12 
   11 
   12 




  408 


    3 
    3 
    3 











    3 


  408 


  410 





 7437 







 7437 





  713 

  713 


  713 



 7437 




 7437 


 7437 















 3491 
 7195 
 7226 
















 7235 














 1109 















 5100 





   48 
   40 







   38 
    2 
   36 
   32 
   38 









   45 

   45 





   45 


   44 




   44 



    1 

   45 












    5 
    5 
    4 





   37 










   36 

   35 



   32 

   32 




   35 
   28 


   35 


    2 





   35 



   29 
   29 

   29 


    3 







   32 

   31 
   31 

   31 
    4 




   29 

   27 

   27 
   27 






   37 


    6 







    9 






    8 

    1 



    9 

    7 


    6 


    6 
    3 






    6 
    4 



    6 











   22 








    1 
    1 


   21 

   21 
   21 

   22 







  180 







 8523 






 8522 
    2 
    1 







   10 






    9 



    8 



    7 





 8523 



  157 
 8522 

  156 



  156 


  156 
    1 

















 6305 






 1144 
 6302 
 6301 



   12 


   10 
   10 
   10 
   10 



    4 



    4 

    4 
    4 

    4 

    4 

    4 
    3 



 6305 




 11196 

 11196 








 6300 

 8473 
    3 




    2 
    2 
    2 





 13197 
 13197 
 13197 

 13197 

 1827 
  381 


  381 





 1827 










 8519 
 8519 



 8519 
 8519 




 8467 










 8523 

 8523 

 8523 
 8522 
   10 


 8522 
 8519 

 6306 
    7 





 6300 


 6300 


 8470 








 6040 

 6040 


 6040 
 6040 

 6040 







  144 








 4300 


 3925 
 4296 
 4296 




   14 
























 6462 


 6462 
 6458 


 5878 
 5788 


    4 


    2 




 4301 




 4301 
 4301 
 4301 
 4189 

  201 




  201 


 4301 








 4301 
    1 




 4301 





 4300 

    2 

    2 


 4300 










 4301 








 6658 























 6462 
 6462 

 6658 

 6657 

 6658 
   11 


 6655 


 6658 
   23 









 5144 
















    1 


    1 





























    1 








    1 



    1 
    1 

    1 











   12 

   12 
   12 






   36 







   34 
   26 


   34 







   14 

   14 


   14 

   12 
    6 



   12 
   12 


    5 
    4 







   12 
    3 
    1 




    2 
















   16 
   36 
   26 




    8 








    7 



    7 












    4 
    1 
    3 


    2 








    4 
    4 


















    2 



    1 





    8 






    3 






    1 








 8531 












 8530 
  138 



 8526 
 8479 


 8525 

  428 

  428 

  427 



   20 




 8407 




 8524 
 7422 




 1651 



 8524 


 8523 



 8523 





 8523 

   21 


 8521 





 8521 
   20 
   20 

 8521 


 8473 






  517 











  516 






  510 










    3 





  510 






   97 



  510 



  106 


  106 



  510 
    3 
  510 
    1 

  510 


   23 

  510 


  145 
  145 

    1 




  510 

  516 
  517 

















   61 







   61 
   61 

   61 


   61 


   61 

   61 








   61 

   61 

   61 



   61 

   15 
   61 











   61 
   15 

   61 
















    3 


    3 































































 4160 

 6499 












    3 




 6500 



    3 
    2 
    3 



    3 


    2 
    1 
    2 
    1 
    2 
    1 
    2 
    3 


 6497 



    1 







    3 
    2 
    3 



    3 


    2 
    1 

    2 
    1 

    2 
    1 

    2 
    3 















  622 
  622 

  622 

  622 




    2 
    2 
    2 











  904 










  145 
  904 









  903 
  904 



  131 
  131 





  812 




  904 


    1 













  903 










  903 









   45 
   44 

  894 



  623 
  397 


    2 

  395 
  895 


  622 







  622 





  622 


  622 

  622 
  342 

  622 








    2 













  881 

  881 






  881 






  881 




  881 













  509 






  837 




  827 

  827 








   45 

   47 
   47 

   47 
  869 










 5959 







 5353 
 5353 



 5959 

    1 

 5959 



 5958 




 5958 



 5352 


 5352 



 5958 
    1 

 5958 









 5958 











 7525 


 7520 



 7513 



 7509 



 7507 


 7506 




 6795 
 7479 


 7462 























 2292 


 2292 
  536 
  536 




 2292 




 2292 


  536 




 2292 












    3 





    4 
    4 




    4 




    4 








    4 


    3 



    3 

    4 







    4 







    2 
    3 

    4 




    4 










    2 





    4 
    4 






    4 



    1 





























    4 




    4 





    4 









    4 



    2 





    4 






























































   61 



   61 






















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * NETLINK      Kernel-user communication protocol.
 *
 *                 Authors:        Alan Cox <alan@lxorguk.ukuu.org.uk>
 *                                 Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
 *                                 Patrick McHardy <kaber@trash.net>
 *
 * Tue Jun 26 14:36:48 MEST 2001 Herbert "herp" Rosmanith
 *                               added netlink_proto_exit
 * Tue Jan 22 18:32:44 BRST 2002 Arnaldo C. de Melo <acme@conectiva.com.br>
 *                                  use nlk_sk, as sk->protinfo is on a diet 8)
 * Fri Jul 22 19:51:12 MEST 2005 Harald Welte <laforge@gnumonks.org>
 *                                  - inc module use count of module that owns
 *                                    the kernel socket in case userspace opens
 *                                    socket of same protocol
 *                                  - remove all module support, since netlink is
 *                                    mandatory if CONFIG_NET=y these days
 */

#include <linux/module.h>

#include <linux/bpf.h>
#include <linux/capability.h>
#include <linux/kernel.h>
#include <linux/filter.h>
#include <linux/init.h>
#include <linux/signal.h>
#include <linux/sched.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/stat.h>
#include <linux/socket.h>
#include <linux/un.h>
#include <linux/fcntl.h>
#include <linux/termios.h>
#include <linux/sockios.h>
#include <linux/net.h>
#include <linux/fs.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/uio.h>
#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/rtnetlink.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/notifier.h>
#include <linux/security.h>
#include <linux/jhash.h>
#include <linux/jiffies.h>
#include <linux/random.h>
#include <linux/bitops.h>
#include <linux/mm.h>
#include <linux/types.h>
#include <linux/audit.h>
#include <linux/mutex.h>
#include <linux/vmalloc.h>
#include <linux/if_arp.h>
#include <linux/rhashtable.h>
#include <asm/cacheflush.h>
#include <linux/hash.h>
#include <linux/net_namespace.h>
#include <linux/nospec.h>
#include <linux/btf_ids.h>

#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <net/sock.h>
#include <net/scm.h>
#include <net/netlink.h>
#define CREATE_TRACE_POINTS
#include <trace/events/netlink.h>

#include "af_netlink.h"
#include "genetlink.h"

struct listeners {
        struct rcu_head                rcu;
        unsigned long                masks[];
};

/* state bits */
#define NETLINK_S_CONGESTED                0x0

static inline int netlink_is_kernel(struct sock *sk)
{
        return nlk_test_bit(KERNEL_SOCKET, sk);
}

struct netlink_table *nl_table __read_mostly;
EXPORT_SYMBOL_GPL(nl_table);

static DECLARE_WAIT_QUEUE_HEAD(nl_table_wait);

static struct lock_class_key nlk_cb_mutex_keys[MAX_LINKS];

static const char *const nlk_cb_mutex_key_strings[MAX_LINKS + 1] = {
        "nlk_cb_mutex-ROUTE",
        "nlk_cb_mutex-1",
        "nlk_cb_mutex-USERSOCK",
        "nlk_cb_mutex-FIREWALL",
        "nlk_cb_mutex-SOCK_DIAG",
        "nlk_cb_mutex-NFLOG",
        "nlk_cb_mutex-XFRM",
        "nlk_cb_mutex-SELINUX",
        "nlk_cb_mutex-ISCSI",
        "nlk_cb_mutex-AUDIT",
        "nlk_cb_mutex-FIB_LOOKUP",
        "nlk_cb_mutex-CONNECTOR",
        "nlk_cb_mutex-NETFILTER",
        "nlk_cb_mutex-IP6_FW",
        "nlk_cb_mutex-DNRTMSG",
        "nlk_cb_mutex-KOBJECT_UEVENT",
        "nlk_cb_mutex-GENERIC",
        "nlk_cb_mutex-17",
        "nlk_cb_mutex-SCSITRANSPORT",
        "nlk_cb_mutex-ECRYPTFS",
        "nlk_cb_mutex-RDMA",
        "nlk_cb_mutex-CRYPTO",
        "nlk_cb_mutex-SMC",
        "nlk_cb_mutex-23",
        "nlk_cb_mutex-24",
        "nlk_cb_mutex-25",
        "nlk_cb_mutex-26",
        "nlk_cb_mutex-27",
        "nlk_cb_mutex-28",
        "nlk_cb_mutex-29",
        "nlk_cb_mutex-30",
        "nlk_cb_mutex-31",
        "nlk_cb_mutex-MAX_LINKS"
};

static int netlink_dump(struct sock *sk, bool lock_taken);

/* nl_table locking explained:
 * Lookup and traversal are protected with an RCU read-side lock. Insertion
 * and removal are protected with per bucket lock while using RCU list
 * modification primitives and may run in parallel to RCU protected lookups.
 * Destruction of the Netlink socket may only occur *after* nl_table_lock has
 * been acquired * either during or after the socket has been removed from
 * the list and after an RCU grace period.
 */
DEFINE_RWLOCK(nl_table_lock);
EXPORT_SYMBOL_GPL(nl_table_lock);
static atomic_t nl_table_users = ATOMIC_INIT(0);

#define nl_deref_protected(X) rcu_dereference_protected(X, lockdep_is_held(&nl_table_lock))

static BLOCKING_NOTIFIER_HEAD(netlink_chain);


static const struct rhashtable_params netlink_rhashtable_params;

void do_trace_netlink_extack(const char *msg)
{
        trace_netlink_extack(msg);
}
EXPORT_SYMBOL(do_trace_netlink_extack);

static inline u32 netlink_group_mask(u32 group)
{
        if (group > 32)
                return 0;
        return group ? 1 << (group - 1) : 0;
}

static struct sk_buff *netlink_to_full_skb(const struct sk_buff *skb,
                                           gfp_t gfp_mask)
{
        unsigned int len = skb->len;
        struct sk_buff *new;

        new = alloc_skb(len, gfp_mask);
        if (new == NULL)
                return NULL;

        NETLINK_CB(new).portid = NETLINK_CB(skb).portid;
        NETLINK_CB(new).dst_group = NETLINK_CB(skb).dst_group;
        NETLINK_CB(new).creds = NETLINK_CB(skb).creds;

        skb_put_data(new, skb->data, len);
        return new;
}

static unsigned int netlink_tap_net_id;

struct netlink_tap_net {
        struct list_head netlink_tap_all;
        struct mutex netlink_tap_lock;
};

int netlink_add_tap(struct netlink_tap *nt)
{
        struct net *net = dev_net(nt->dev);
        struct netlink_tap_net *nn = net_generic(net, netlink_tap_net_id);

        if (unlikely(nt->dev->type != ARPHRD_NETLINK))
                return -EINVAL;

        mutex_lock(&nn->netlink_tap_lock);
        list_add_rcu(&nt->list, &nn->netlink_tap_all);
        mutex_unlock(&nn->netlink_tap_lock);

        __module_get(nt->module);

        return 0;
}
EXPORT_SYMBOL_GPL(netlink_add_tap);

static int __netlink_remove_tap(struct netlink_tap *nt)
{
        struct net *net = dev_net(nt->dev);
        struct netlink_tap_net *nn = net_generic(net, netlink_tap_net_id);
        bool found = false;
        struct netlink_tap *tmp;

        mutex_lock(&nn->netlink_tap_lock);

        list_for_each_entry(tmp, &nn->netlink_tap_all, list) {
                if (nt == tmp) {
                        list_del_rcu(&nt->list);
                        found = true;
                        goto out;
                }
        }

        pr_warn("__netlink_remove_tap: %p not found\n", nt);
out:
        mutex_unlock(&nn->netlink_tap_lock);

        if (found)
                module_put(nt->module);

        return found ? 0 : -ENODEV;
}

int netlink_remove_tap(struct netlink_tap *nt)
{
        int ret;

        ret = __netlink_remove_tap(nt);
        synchronize_net();

        return ret;
}
EXPORT_SYMBOL_GPL(netlink_remove_tap);

static __net_init int netlink_tap_init_net(struct net *net)
{
        struct netlink_tap_net *nn = net_generic(net, netlink_tap_net_id);

        INIT_LIST_HEAD(&nn->netlink_tap_all);
        mutex_init(&nn->netlink_tap_lock);
        return 0;
}

static struct pernet_operations netlink_tap_net_ops = {
        .init = netlink_tap_init_net,
        .id   = &netlink_tap_net_id,
        .size = sizeof(struct netlink_tap_net),
};

static bool netlink_filter_tap(const struct sk_buff *skb)
{
        struct sock *sk = skb->sk;

        /* We take the more conservative approach and
         * whitelist socket protocols that may pass.
         */
        switch (sk->sk_protocol) {
        case NETLINK_ROUTE:
        case NETLINK_USERSOCK:
        case NETLINK_SOCK_DIAG:
        case NETLINK_NFLOG:
        case NETLINK_XFRM:
        case NETLINK_FIB_LOOKUP:
        case NETLINK_NETFILTER:
        case NETLINK_GENERIC:
                return true;
        }

        return false;
}

static int __netlink_deliver_tap_skb(struct sk_buff *skb,
                                     struct net_device *dev)
{
        struct sk_buff *nskb;
        struct sock *sk = skb->sk;
        int ret = -ENOMEM;

        if (!net_eq(dev_net(dev), sock_net(sk)))
                return 0;

        dev_hold(dev);

        if (is_vmalloc_addr(skb->head))
                nskb = netlink_to_full_skb(skb, GFP_ATOMIC);
        else
                nskb = skb_clone(skb, GFP_ATOMIC);
        if (nskb) {
                nskb->dev = dev;
                nskb->protocol = htons((u16) sk->sk_protocol);
                nskb->pkt_type = netlink_is_kernel(sk) ?
                                 PACKET_KERNEL : PACKET_USER;
                skb_reset_network_header(nskb);
                ret = dev_queue_xmit(nskb);
                if (unlikely(ret > 0))
                        ret = net_xmit_errno(ret);
        }

        dev_put(dev);
        return ret;
}

static void __netlink_deliver_tap(struct sk_buff *skb, struct netlink_tap_net *nn)
{
        int ret;
        struct netlink_tap *tmp;

        if (!netlink_filter_tap(skb))
                return;

        list_for_each_entry_rcu(tmp, &nn->netlink_tap_all, list) {
                ret = __netlink_deliver_tap_skb(skb, tmp->dev);
                if (unlikely(ret))
                        break;
        }
}

static void netlink_deliver_tap(struct net *net, struct sk_buff *skb)
{
        struct netlink_tap_net *nn = net_generic(net, netlink_tap_net_id);

        rcu_read_lock();

        if (unlikely(!list_empty(&nn->netlink_tap_all)))
                __netlink_deliver_tap(skb, nn);

        rcu_read_unlock();
}

static void netlink_deliver_tap_kernel(struct sock *dst, struct sock *src,
                                       struct sk_buff *skb)
{
        if (!(netlink_is_kernel(dst) && netlink_is_kernel(src)))
                netlink_deliver_tap(sock_net(dst), skb);
}

static void netlink_overrun(struct sock *sk)
{
        if (!nlk_test_bit(RECV_NO_ENOBUFS, sk)) {
                if (!test_and_set_bit(NETLINK_S_CONGESTED,
                                      &nlk_sk(sk)->state)) {
                        WRITE_ONCE(sk->sk_err, ENOBUFS);
                        sk_error_report(sk);
                }
        }
        sk_drops_inc(sk);
}

static void netlink_rcv_wake(struct sock *sk)
{
        struct netlink_sock *nlk = nlk_sk(sk);

        if (skb_queue_empty_lockless(&sk->sk_receive_queue))
                clear_bit(NETLINK_S_CONGESTED, &nlk->state);
        if (!test_bit(NETLINK_S_CONGESTED, &nlk->state))
                wake_up_interruptible(&nlk->wait);
}

static void netlink_skb_destructor(struct sk_buff *skb)
{
        if (is_vmalloc_addr(skb->head)) {
                if (!skb->cloned ||
                    !atomic_dec_return(&(skb_shinfo(skb)->dataref)))
                        vfree_atomic(skb->head);

                skb->head = NULL;
        }
        if (skb->sk != NULL)
                sock_rfree(skb);
}

static void netlink_skb_set_owner_r(struct sk_buff *skb, struct sock *sk)
{
        WARN_ON(skb->sk != NULL);
        skb->sk = sk;
        skb->destructor = netlink_skb_destructor;
        sk_mem_charge(sk, skb->truesize);
}

static void netlink_sock_destruct(struct sock *sk)
{
        skb_queue_purge(&sk->sk_receive_queue);

        if (!sock_flag(sk, SOCK_DEAD)) {
                printk(KERN_ERR "Freeing alive netlink socket %p\n", sk);
                return;
        }

        WARN_ON(atomic_read(&sk->sk_rmem_alloc));
        WARN_ON(refcount_read(&sk->sk_wmem_alloc));
        WARN_ON(nlk_sk(sk)->groups);
}

/* This lock without WQ_FLAG_EXCLUSIVE is good on UP and it is _very_ bad on
 * SMP. Look, when several writers sleep and reader wakes them up, all but one
 * immediately hit write lock and grab all the cpus. Exclusive sleep solves
 * this, _but_ remember, it adds useless work on UP machines.
 */

void netlink_table_grab(void)
        __acquires(nl_table_lock)
{
        might_sleep();

        write_lock_irq(&nl_table_lock);

        if (atomic_read(&nl_table_users)) {
                DECLARE_WAITQUEUE(wait, current);

                add_wait_queue_exclusive(&nl_table_wait, &wait);
                for (;;) {
                        set_current_state(TASK_UNINTERRUPTIBLE);
                        if (atomic_read(&nl_table_users) == 0)
                                break;
                        write_unlock_irq(&nl_table_lock);
                        schedule();
                        write_lock_irq(&nl_table_lock);
                }

                __set_current_state(TASK_RUNNING);
                remove_wait_queue(&nl_table_wait, &wait);
        }
}

void netlink_table_ungrab(void)
        __releases(nl_table_lock)
{
        write_unlock_irq(&nl_table_lock);
        wake_up(&nl_table_wait);
}

static inline void
netlink_lock_table(void)
{
        unsigned long flags;

        /* read_lock() synchronizes us to netlink_table_grab */

        read_lock_irqsave(&nl_table_lock, flags);
        atomic_inc(&nl_table_users);
        read_unlock_irqrestore(&nl_table_lock, flags);
}

static inline void
netlink_unlock_table(void)
{
        if (atomic_dec_and_test(&nl_table_users))
                wake_up(&nl_table_wait);
}

struct netlink_compare_arg
{
        possible_net_t pnet;
        u32 portid;
};

/* Doing sizeof directly may yield 4 extra bytes on 64-bit. */
#define netlink_compare_arg_len \
        (offsetof(struct netlink_compare_arg, portid) + sizeof(u32))

static inline int netlink_compare(struct rhashtable_compare_arg *arg,
                                  const void *ptr)
{
        const struct netlink_compare_arg *x = arg->key;
        const struct netlink_sock *nlk = ptr;

        return nlk->portid != x->portid ||
               !net_eq(sock_net(&nlk->sk), read_pnet(&x->pnet));
}

static void netlink_compare_arg_init(struct netlink_compare_arg *arg,
                                     struct net *net, u32 portid)
{
        memset(arg, 0, sizeof(*arg));
        write_pnet(&arg->pnet, net);
        arg->portid = portid;
}

static struct sock *__netlink_lookup(struct netlink_table *table, u32 portid,
                                     struct net *net)
{
        struct netlink_compare_arg arg;

        netlink_compare_arg_init(&arg, net, portid);
        return rhashtable_lookup_fast(&table->hash, &arg,
                                      netlink_rhashtable_params);
}

static int __netlink_insert(struct netlink_table *table, struct sock *sk)
{
        struct netlink_compare_arg arg;

        netlink_compare_arg_init(&arg, sock_net(sk), nlk_sk(sk)->portid);
        return rhashtable_lookup_insert_key(&table->hash, &arg,
                                            &nlk_sk(sk)->node,
                                            netlink_rhashtable_params);
}

static struct sock *netlink_lookup(struct net *net, int protocol, u32 portid)
{
        struct netlink_table *table = &nl_table[protocol];
        struct sock *sk;

        rcu_read_lock();
        sk = __netlink_lookup(table, portid, net);
        if (sk)
                sock_hold(sk);
        rcu_read_unlock();

        return sk;
}

static const struct proto_ops netlink_ops;

static void
netlink_update_listeners(struct sock *sk)
{
        struct netlink_table *tbl = &nl_table[sk->sk_protocol];
        unsigned long mask;
        unsigned int i;
        struct listeners *listeners;

        listeners = nl_deref_protected(tbl->listeners);
        if (!listeners)
                return;

        for (i = 0; i < NLGRPLONGS(tbl->groups); i++) {
                mask = 0;
                sk_for_each_bound(sk, &tbl->mc_list) {
                        if (i < NLGRPLONGS(nlk_sk(sk)->ngroups))
                                mask |= nlk_sk(sk)->groups[i];
                }
                listeners->masks[i] = mask;
        }
        /* this function is only called with the netlink table "grabbed", which
         * makes sure updates are visible before bind or setsockopt return. */
}

static int netlink_insert(struct sock *sk, u32 portid)
{
        struct netlink_table *table = &nl_table[sk->sk_protocol];
        int err;

        lock_sock(sk);

        err = nlk_sk(sk)->portid == portid ? 0 : -EBUSY;
        if (nlk_sk(sk)->bound)
                goto err;

        /* portid can be read locklessly from netlink_getname(). */
        WRITE_ONCE(nlk_sk(sk)->portid, portid);

        sock_hold(sk);

        err = __netlink_insert(table, sk);
        if (err) {
                /* In case the hashtable backend returns with -EBUSY
                 * from here, it must not escape to the caller.
                 */
                if (unlikely(err == -EBUSY))
                        err = -EOVERFLOW;
                if (err == -EEXIST)
                        err = -EADDRINUSE;
                sock_put(sk);
                goto err;
        }

        /* We need to ensure that the socket is hashed and visible. */
        smp_wmb();
        /* Paired with lockless reads from netlink_bind(),
         * netlink_connect() and netlink_sendmsg().
         */
        WRITE_ONCE(nlk_sk(sk)->bound, portid);

err:
        release_sock(sk);
        return err;
}

static void netlink_remove(struct sock *sk)
{
        struct netlink_table *table;

        table = &nl_table[sk->sk_protocol];
        if (!rhashtable_remove_fast(&table->hash, &nlk_sk(sk)->node,
                                    netlink_rhashtable_params))
                __sock_put(sk);

        netlink_table_grab();
        if (nlk_sk(sk)->subscriptions) {
                __sk_del_bind_node(sk);
                netlink_update_listeners(sk);
        }
        if (sk->sk_protocol == NETLINK_GENERIC)
                atomic_inc(&genl_sk_destructing_cnt);
        netlink_table_ungrab();
}

static struct proto netlink_proto = {
        .name          = "NETLINK",
        .owner          = THIS_MODULE,
        .obj_size = sizeof(struct netlink_sock),
};

static int __netlink_create(struct net *net, struct socket *sock,
                            int protocol, int kern)
{
        struct sock *sk;
        struct netlink_sock *nlk;

        sock->ops = &netlink_ops;

        sk = sk_alloc(net, PF_NETLINK, GFP_KERNEL, &netlink_proto, kern);
        if (!sk)
                return -ENOMEM;

        sock_init_data(sock, sk);

        nlk = nlk_sk(sk);
        mutex_init(&nlk->nl_cb_mutex);
        lockdep_set_class_and_name(&nlk->nl_cb_mutex,
                                           nlk_cb_mutex_keys + protocol,
                                           nlk_cb_mutex_key_strings[protocol]);
        init_waitqueue_head(&nlk->wait);

        sk->sk_destruct = netlink_sock_destruct;
        sk->sk_protocol = protocol;
        return 0;
}

static int netlink_create(struct net *net, struct socket *sock, int protocol,
                          int kern)
{
        struct module *module = NULL;
        struct netlink_sock *nlk;
        int (*bind)(struct net *net, int group);
        void (*unbind)(struct net *net, int group);
        void (*release)(struct sock *sock, unsigned long *groups);
        int err = 0;

        sock->state = SS_UNCONNECTED;

        if (sock->type != SOCK_RAW && sock->type != SOCK_DGRAM)
                return -ESOCKTNOSUPPORT;

        if (protocol < 0 || protocol >= MAX_LINKS)
                return -EPROTONOSUPPORT;
        protocol = array_index_nospec(protocol, MAX_LINKS);

        netlink_lock_table();
#ifdef CONFIG_MODULES
        if (!nl_table[protocol].registered) {
                netlink_unlock_table();
                request_module("net-pf-%d-proto-%d", PF_NETLINK, protocol);
                netlink_lock_table();
        }
#endif
        if (nl_table[protocol].registered &&
            try_module_get(nl_table[protocol].module))
                module = nl_table[protocol].module;
        else
                err = -EPROTONOSUPPORT;
        bind = nl_table[protocol].bind;
        unbind = nl_table[protocol].unbind;
        release = nl_table[protocol].release;
        netlink_unlock_table();

        if (err < 0)
                goto out;

        err = __netlink_create(net, sock, protocol, kern);
        if (err < 0)
                goto out_module;

        sock_prot_inuse_add(net, &netlink_proto, 1);

        nlk = nlk_sk(sock->sk);
        nlk->module = module;
        nlk->netlink_bind = bind;
        nlk->netlink_unbind = unbind;
        nlk->netlink_release = release;
out:
        return err;

out_module:
        module_put(module);
        goto out;
}

static void deferred_put_nlk_sk(struct rcu_head *head)
{
        struct netlink_sock *nlk = container_of(head, struct netlink_sock, rcu);
        struct sock *sk = &nlk->sk;

        kfree(nlk->groups);
        nlk->groups = NULL;

        if (!refcount_dec_and_test(&sk->sk_refcnt))
                return;

        sk_free(sk);
}

static int netlink_release(struct socket *sock)
{
        struct sock *sk = sock->sk;
        struct netlink_sock *nlk;

        if (!sk)
                return 0;

        netlink_remove(sk);
        sock_orphan(sk);
        nlk = nlk_sk(sk);

        /*
         * OK. Socket is unlinked, any packets that arrive now
         * will be purged.
         */
        if (nlk->netlink_release)
                nlk->netlink_release(sk, nlk->groups);

        /* must not acquire netlink_table_lock in any way again before unbind
         * and notifying genetlink is done as otherwise it might deadlock
         */
        if (nlk->netlink_unbind) {
                int i;

                for (i = 0; i < nlk->ngroups; i++)
                        if (test_bit(i, nlk->groups))
                                nlk->netlink_unbind(sock_net(sk), i + 1);
        }
        if (sk->sk_protocol == NETLINK_GENERIC &&
            atomic_dec_return(&genl_sk_destructing_cnt) == 0)
                wake_up(&genl_sk_destructing_waitq);

        sock->sk = NULL;
        wake_up_interruptible_all(&nlk->wait);

        skb_queue_purge(&sk->sk_write_queue);

        if (nlk->portid && nlk->bound) {
                struct netlink_notify n = {
                                                .net = sock_net(sk),
                                                .protocol = sk->sk_protocol,
                                                .portid = nlk->portid,
                                          };
                blocking_notifier_call_chain(&netlink_chain,
                                NETLINK_URELEASE, &n);
        }

        /* Terminate any outstanding dump */
        if (nlk->cb_running) {
                if (nlk->cb.done)
                        nlk->cb.done(&nlk->cb);
                module_put(nlk->cb.module);
                kfree_skb(nlk->cb.skb);
                WRITE_ONCE(nlk->cb_running, false);
        }

        module_put(nlk->module);

        if (netlink_is_kernel(sk)) {
                netlink_table_grab();
                BUG_ON(nl_table[sk->sk_protocol].registered == 0);
                if (--nl_table[sk->sk_protocol].registered == 0) {
                        struct listeners *old;

                        old = nl_deref_protected(nl_table[sk->sk_protocol].listeners);
                        RCU_INIT_POINTER(nl_table[sk->sk_protocol].listeners, NULL);
                        kfree_rcu(old, rcu);
                        nl_table[sk->sk_protocol].module = NULL;
                        nl_table[sk->sk_protocol].bind = NULL;
                        nl_table[sk->sk_protocol].unbind = NULL;
                        nl_table[sk->sk_protocol].flags = 0;
                        nl_table[sk->sk_protocol].registered = 0;
                }
                netlink_table_ungrab();
        }

        sock_prot_inuse_add(sock_net(sk), &netlink_proto, -1);

        call_rcu(&nlk->rcu, deferred_put_nlk_sk);
        return 0;
}

static int netlink_autobind(struct socket *sock)
{
        struct sock *sk = sock->sk;
        struct net *net = sock_net(sk);
        struct netlink_table *table = &nl_table[sk->sk_protocol];
        s32 portid = task_tgid_vnr(current);
        int err;
        s32 rover = -4096;
        bool ok;

retry:
        cond_resched();
        rcu_read_lock();
        ok = !__netlink_lookup(table, portid, net);
        rcu_read_unlock();
        if (!ok) {
                /* Bind collision, search negative portid values. */
                if (rover == -4096)
                        /* rover will be in range [S32_MIN, -4097] */
                        rover = S32_MIN + get_random_u32_below(-4096 - S32_MIN);
                else if (rover >= -4096)
                        rover = -4097;
                portid = rover--;
                goto retry;
        }

        err = netlink_insert(sk, portid);
        if (err == -EADDRINUSE)
                goto retry;

        /* If 2 threads race to autobind, that is fine.  */
        if (err == -EBUSY)
                err = 0;

        return err;
}

/**
 * __netlink_ns_capable - General netlink message capability test
 * @nsp: NETLINK_CB of the socket buffer holding a netlink command from userspace.
 * @user_ns: The user namespace of the capability to use
 * @cap: The capability to use
 *
 * Test to see if the opener of the socket we received the message
 * from had when the netlink socket was created and the sender of the
 * message has the capability @cap in the user namespace @user_ns.
 */
bool __netlink_ns_capable(const struct netlink_skb_parms *nsp,
                        struct user_namespace *user_ns, int cap)
{
        return ((nsp->flags & NETLINK_SKB_DST) ||
                file_ns_capable(nsp->sk->sk_socket->file, user_ns, cap)) &&
                ns_capable(user_ns, cap);
}
EXPORT_SYMBOL(__netlink_ns_capable);

/**
 * netlink_ns_capable - General netlink message capability test
 * @skb: socket buffer holding a netlink command from userspace
 * @user_ns: The user namespace of the capability to use
 * @cap: The capability to use
 *
 * Test to see if the opener of the socket we received the message
 * from had when the netlink socket was created and the sender of the
 * message has the capability @cap in the user namespace @user_ns.
 */
bool netlink_ns_capable(const struct sk_buff *skb,
                        struct user_namespace *user_ns, int cap)
{
        return __netlink_ns_capable(&NETLINK_CB(skb), user_ns, cap);
}
EXPORT_SYMBOL(netlink_ns_capable);

/**
 * netlink_capable - Netlink global message capability test
 * @skb: socket buffer holding a netlink command from userspace
 * @cap: The capability to use
 *
 * Test to see if the opener of the socket we received the message
 * from had when the netlink socket was created and the sender of the
 * message has the capability @cap in all user namespaces.
 */
bool netlink_capable(const struct sk_buff *skb, int cap)
{
        return netlink_ns_capable(skb, &init_user_ns, cap);
}
EXPORT_SYMBOL(netlink_capable);

/**
 * netlink_net_capable - Netlink network namespace message capability test
 * @skb: socket buffer holding a netlink command from userspace
 * @cap: The capability to use
 *
 * Test to see if the opener of the socket we received the message
 * from had when the netlink socket was created and the sender of the
 * message has the capability @cap over the network namespace of
 * the socket we received the message from.
 */
bool netlink_net_capable(const struct sk_buff *skb, int cap)
{
        return netlink_ns_capable(skb, sock_net(skb->sk)->user_ns, cap);
}
EXPORT_SYMBOL(netlink_net_capable);

static inline int netlink_allowed(const struct socket *sock, unsigned int flag)
{
        return (nl_table[sock->sk->sk_protocol].flags & flag) ||
                ns_capable(sock_net(sock->sk)->user_ns, CAP_NET_ADMIN);
}

static void
netlink_update_subscriptions(struct sock *sk, unsigned int subscriptions)
{
        struct netlink_sock *nlk = nlk_sk(sk);

        if (nlk->subscriptions && !subscriptions)
                __sk_del_bind_node(sk);
        else if (!nlk->subscriptions && subscriptions)
                sk_add_bind_node(sk, &nl_table[sk->sk_protocol].mc_list);
        nlk->subscriptions = subscriptions;
}

static int netlink_realloc_groups(struct sock *sk)
{
        struct netlink_sock *nlk = nlk_sk(sk);
        unsigned int groups;
        unsigned long *new_groups;
        int err = 0;

        netlink_table_grab();

        groups = nl_table[sk->sk_protocol].groups;
        if (!nl_table[sk->sk_protocol].registered) {
                err = -ENOENT;
                goto out_unlock;
        }

        if (nlk->ngroups >= groups)
                goto out_unlock;

        new_groups = krealloc(nlk->groups, NLGRPSZ(groups), GFP_ATOMIC);
        if (new_groups == NULL) {
                err = -ENOMEM;
                goto out_unlock;
        }
        memset((char *)new_groups + NLGRPSZ(nlk->ngroups), 0,
               NLGRPSZ(groups) - NLGRPSZ(nlk->ngroups));

        nlk->groups = new_groups;
        nlk->ngroups = groups;
 out_unlock:
        netlink_table_ungrab();
        return err;
}

static void netlink_undo_bind(int group, long unsigned int groups,
                              struct sock *sk)
{
        struct netlink_sock *nlk = nlk_sk(sk);
        int undo;

        if (!nlk->netlink_unbind)
                return;

        for (undo = 0; undo < group; undo++)
                if (test_bit(undo, &groups))
                        nlk->netlink_unbind(sock_net(sk), undo + 1);
}

static int netlink_bind(struct socket *sock, struct sockaddr_unsized *addr,
                        int addr_len)
{
        struct sock *sk = sock->sk;
        struct net *net = sock_net(sk);
        struct netlink_sock *nlk = nlk_sk(sk);
        struct sockaddr_nl *nladdr = (struct sockaddr_nl *)addr;
        int err = 0;
        unsigned long groups;
        bool bound;

        if (addr_len < sizeof(struct sockaddr_nl))
                return -EINVAL;

        if (nladdr->nl_family != AF_NETLINK)
                return -EINVAL;
        groups = nladdr->nl_groups;

        /* Only superuser is allowed to listen multicasts */
        if (groups) {
                if (!netlink_allowed(sock, NL_CFG_F_NONROOT_RECV))
                        return -EPERM;
                err = netlink_realloc_groups(sk);
                if (err)
                        return err;
        }

        if (nlk->ngroups < BITS_PER_LONG)
                groups &= (1UL << nlk->ngroups) - 1;

        /* Paired with WRITE_ONCE() in netlink_insert() */
        bound = READ_ONCE(nlk->bound);
        if (bound) {
                /* Ensure nlk->portid is up-to-date. */
                smp_rmb();

                if (nladdr->nl_pid != nlk->portid)
                        return -EINVAL;
        }

        if (nlk->netlink_bind && groups) {
                int group;

                /* nl_groups is a u32, so cap the maximum groups we can bind */
                for (group = 0; group < BITS_PER_TYPE(u32); group++) {
                        if (!test_bit(group, &groups))
                                continue;
                        err = nlk->netlink_bind(net, group + 1);
                        if (!err)
                                continue;
                        netlink_undo_bind(group, groups, sk);
                        return err;
                }
        }

        /* No need for barriers here as we return to user-space without
         * using any of the bound attributes.
         */
        netlink_lock_table();
        if (!bound) {
                err = nladdr->nl_pid ?
                        netlink_insert(sk, nladdr->nl_pid) :
                        netlink_autobind(sock);
                if (err) {
                        netlink_undo_bind(BITS_PER_TYPE(u32), groups, sk);
                        goto unlock;
                }
        }

        if (!groups && (nlk->groups == NULL || !(u32)nlk->groups[0]))
                goto unlock;
        netlink_unlock_table();

        netlink_table_grab();
        netlink_update_subscriptions(sk, nlk->subscriptions +
                                         hweight32(groups) -
                                         hweight32(nlk->groups[0]));
        nlk->groups[0] = (nlk->groups[0] & ~0xffffffffUL) | groups;
        netlink_update_listeners(sk);
        netlink_table_ungrab();

        return 0;

unlock:
        netlink_unlock_table();
        return err;
}

static int netlink_connect(struct socket *sock, struct sockaddr_unsized *addr,
                           int alen, int flags)
{
        int err = 0;
        struct sock *sk = sock->sk;
        struct netlink_sock *nlk = nlk_sk(sk);
        struct sockaddr_nl *nladdr = (struct sockaddr_nl *)addr;

        if (alen < sizeof(addr->sa_family))
                return -EINVAL;

        if (addr->sa_family == AF_UNSPEC) {
                /* paired with READ_ONCE() in netlink_getsockbyportid() */
                WRITE_ONCE(sk->sk_state, NETLINK_UNCONNECTED);
                /* dst_portid and dst_group can be read locklessly */
                WRITE_ONCE(nlk->dst_portid, 0);
                WRITE_ONCE(nlk->dst_group, 0);
                return 0;
        }
        if (addr->sa_family != AF_NETLINK)
                return -EINVAL;

        if (alen < sizeof(struct sockaddr_nl))
                return -EINVAL;

        if ((nladdr->nl_groups || nladdr->nl_pid) &&
            !netlink_allowed(sock, NL_CFG_F_NONROOT_SEND))
                return -EPERM;

        /* No need for barriers here as we return to user-space without
         * using any of the bound attributes.
         * Paired with WRITE_ONCE() in netlink_insert().
         */
        if (!READ_ONCE(nlk->bound))
                err = netlink_autobind(sock);

        if (err == 0) {
                /* paired with READ_ONCE() in netlink_getsockbyportid() */
                WRITE_ONCE(sk->sk_state, NETLINK_CONNECTED);
                /* dst_portid and dst_group can be read locklessly */
                WRITE_ONCE(nlk->dst_portid, nladdr->nl_pid);
                WRITE_ONCE(nlk->dst_group, ffs(nladdr->nl_groups));
        }

        return err;
}

static int netlink_getname(struct socket *sock, struct sockaddr *addr,
                           int peer)
{
        struct sock *sk = sock->sk;
        struct netlink_sock *nlk = nlk_sk(sk);
        DECLARE_SOCKADDR(struct sockaddr_nl *, nladdr, addr);

        nladdr->nl_family = AF_NETLINK;
        nladdr->nl_pad = 0;

        if (peer) {
                /* Paired with WRITE_ONCE() in netlink_connect() */
                nladdr->nl_pid = READ_ONCE(nlk->dst_portid);
                nladdr->nl_groups = netlink_group_mask(READ_ONCE(nlk->dst_group));
        } else {
                /* Paired with WRITE_ONCE() in netlink_insert() */
                nladdr->nl_pid = READ_ONCE(nlk->portid);
                netlink_lock_table();
                nladdr->nl_groups = nlk->groups ? nlk->groups[0] : 0;
                netlink_unlock_table();
        }
        return sizeof(*nladdr);
}

static int netlink_ioctl(struct socket *sock, unsigned int cmd,
                         unsigned long arg)
{
        /* try to hand this ioctl down to the NIC drivers.
         */
        return -ENOIOCTLCMD;
}

static struct sock *netlink_getsockbyportid(struct sock *ssk, u32 portid)
{
        struct sock *sock;
        struct netlink_sock *nlk;

        sock = netlink_lookup(sock_net(ssk), ssk->sk_protocol, portid);
        if (!sock)
                return ERR_PTR(-ECONNREFUSED);

        /* Don't bother queuing skb if kernel socket has no input function */
        nlk = nlk_sk(sock);
        /* dst_portid and sk_state can be changed in netlink_connect() */
        if (READ_ONCE(sock->sk_state) == NETLINK_CONNECTED &&
            READ_ONCE(nlk->dst_portid) != nlk_sk(ssk)->portid) {
                sock_put(sock);
                return ERR_PTR(-ECONNREFUSED);
        }
        return sock;
}

struct sock *netlink_getsockbyfd(int fd)
{
        CLASS(fd, f)(fd);
        struct inode *inode;
        struct sock *sock;

        if (fd_empty(f))
                return ERR_PTR(-EBADF);

        inode = file_inode(fd_file(f));
        if (!S_ISSOCK(inode->i_mode))
                return ERR_PTR(-ENOTSOCK);

        sock = SOCKET_I(inode)->sk;
        if (sock->sk_family != AF_NETLINK)
                return ERR_PTR(-EINVAL);

        sock_hold(sock);
        return sock;
}

struct sk_buff *netlink_alloc_large_skb(unsigned int size, int broadcast)
{
        size_t head_size = SKB_HEAD_ALIGN(size);
        struct sk_buff *skb;
        void *data;

        if (head_size <= PAGE_SIZE || broadcast)
                return alloc_skb(size, GFP_KERNEL);

        data = kvmalloc(head_size, GFP_KERNEL);
        if (!data)
                return NULL;

        skb = __build_skb(data, head_size);
        if (!skb)
                kvfree(data);
        else if (is_vmalloc_addr(data))
                skb->destructor = netlink_skb_destructor;

        return skb;
}

/*
 * Attach a skb to a netlink socket.
 * The caller must hold a reference to the destination socket. On error, the
 * reference is dropped. The skb is not send to the destination, just all
 * all error checks are performed and memory in the queue is reserved.
 * Return values:
 * < 0: error. skb freed, reference to sock dropped.
 * 0: continue
 * 1: repeat lookup - reference dropped while waiting for socket memory.
 */
int netlink_attachskb(struct sock *sk, struct sk_buff *skb,
                      long *timeo, struct sock *ssk)
{
        DECLARE_WAITQUEUE(wait, current);
        struct netlink_sock *nlk;
        unsigned int rmem;

        nlk = nlk_sk(sk);
        rmem = atomic_add_return(skb->truesize, &sk->sk_rmem_alloc);

        if ((rmem == skb->truesize || rmem <= READ_ONCE(sk->sk_rcvbuf)) &&
            !test_bit(NETLINK_S_CONGESTED, &nlk->state)) {
                netlink_skb_set_owner_r(skb, sk);
                return 0;
        }

        atomic_sub(skb->truesize, &sk->sk_rmem_alloc);

        if (!*timeo) {
                if (!ssk || netlink_is_kernel(ssk))
                        netlink_overrun(sk);
                sock_put(sk);
                kfree_skb(skb);
                return -EAGAIN;
        }

        __set_current_state(TASK_INTERRUPTIBLE);
        add_wait_queue(&nlk->wait, &wait);
        rmem = atomic_read(&sk->sk_rmem_alloc);

        if (((rmem && rmem + skb->truesize > READ_ONCE(sk->sk_rcvbuf)) ||
             test_bit(NETLINK_S_CONGESTED, &nlk->state)) &&
            !sock_flag(sk, SOCK_DEAD))
                *timeo = schedule_timeout(*timeo);

        __set_current_state(TASK_RUNNING);
        remove_wait_queue(&nlk->wait, &wait);
        sock_put(sk);

        if (signal_pending(current)) {
                kfree_skb(skb);
                return sock_intr_errno(*timeo);
        }

        return 1;
}

static int __netlink_sendskb(struct sock *sk, struct sk_buff *skb)
{
        int len = skb->len;

        netlink_deliver_tap(sock_net(sk), skb);

        skb_queue_tail(&sk->sk_receive_queue, skb);
        sk->sk_data_ready(sk);
        return len;
}

int netlink_sendskb(struct sock *sk, struct sk_buff *skb)
{
        int len = __netlink_sendskb(sk, skb);

        sock_put(sk);
        return len;
}

void netlink_detachskb(struct sock *sk, struct sk_buff *skb)
{
        kfree_skb(skb);
        sock_put(sk);
}

static struct sk_buff *netlink_trim(struct sk_buff *skb, gfp_t allocation)
{
        int delta;

        skb_assert_len(skb);
        WARN_ON(skb->sk != NULL);
        delta = skb->end - skb->tail;
        if (is_vmalloc_addr(skb->head) || delta * 2 < skb->truesize)
                return skb;

        if (skb_shared(skb)) {
                struct sk_buff *nskb = skb_clone(skb, allocation);
                if (!nskb)
                        return skb;
                consume_skb(skb);
                skb = nskb;
        }

        pskb_expand_head(skb, 0, -delta,
                         (allocation & ~__GFP_DIRECT_RECLAIM) |
                         __GFP_NOWARN | __GFP_NORETRY);
        return skb;
}

static int netlink_unicast_kernel(struct sock *sk, struct sk_buff *skb,
                                  struct sock *ssk)
{
        int ret;
        struct netlink_sock *nlk = nlk_sk(sk);

        ret = -ECONNREFUSED;
        if (nlk->netlink_rcv != NULL) {
                ret = skb->len;
                atomic_add(skb->truesize, &sk->sk_rmem_alloc);
                netlink_skb_set_owner_r(skb, sk);
                NETLINK_CB(skb).sk = ssk;
                netlink_deliver_tap_kernel(sk, ssk, skb);
                nlk->netlink_rcv(skb);
                consume_skb(skb);
        } else {
                kfree_skb(skb);
        }
        sock_put(sk);
        return ret;
}

int netlink_unicast(struct sock *ssk, struct sk_buff *skb,
                    u32 portid, int nonblock)
{
        struct sock *sk;
        int err;
        long timeo;

        skb = netlink_trim(skb, gfp_any());

        timeo = sock_sndtimeo(ssk, nonblock);
retry:
        sk = netlink_getsockbyportid(ssk, portid);
        if (IS_ERR(sk)) {
                kfree_skb(skb);
                return PTR_ERR(sk);
        }
        if (netlink_is_kernel(sk))
                return netlink_unicast_kernel(sk, skb, ssk);

        if (sk_filter(sk, skb)) {
                err = skb->len;
                kfree_skb(skb);
                sock_put(sk);
                return err;
        }

        err = netlink_attachskb(sk, skb, &timeo, ssk);
        if (err == 1)
                goto retry;
        if (err)
                return err;

        return netlink_sendskb(sk, skb);
}
EXPORT_SYMBOL(netlink_unicast);

int netlink_has_listeners(struct sock *sk, unsigned int group)
{
        int res = 0;
        struct listeners *listeners;

        BUG_ON(!netlink_is_kernel(sk));

        rcu_read_lock();
        listeners = rcu_dereference(nl_table[sk->sk_protocol].listeners);

        if (listeners && group - 1 < nl_table[sk->sk_protocol].groups)
                res = test_bit(group - 1, listeners->masks);

        rcu_read_unlock();

        return res;
}
EXPORT_SYMBOL_GPL(netlink_has_listeners);

bool netlink_strict_get_check(struct sk_buff *skb)
{
        return nlk_test_bit(STRICT_CHK, NETLINK_CB(skb).sk);
}
EXPORT_SYMBOL_GPL(netlink_strict_get_check);

static int netlink_broadcast_deliver(struct sock *sk, struct sk_buff *skb)
{
        struct netlink_sock *nlk = nlk_sk(sk);
        unsigned int rmem, rcvbuf;

        rmem = atomic_add_return(skb->truesize, &sk->sk_rmem_alloc);
        rcvbuf = READ_ONCE(sk->sk_rcvbuf);

        if ((rmem == skb->truesize || rmem <= rcvbuf) &&
            !test_bit(NETLINK_S_CONGESTED, &nlk->state)) {
                netlink_skb_set_owner_r(skb, sk);
                __netlink_sendskb(sk, skb);
                return rmem > (rcvbuf >> 1);
        }

        atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
        return -1;
}

struct netlink_broadcast_data {
        struct sock *exclude_sk;
        struct net *net;
        u32 portid;
        u32 group;
        int failure;
        int delivery_failure;
        int congested;
        int delivered;
        gfp_t allocation;
        struct sk_buff *skb, *skb2;
        int (*tx_filter)(struct sock *dsk, struct sk_buff *skb, void *data);
        void *tx_data;
};

static void do_one_broadcast(struct sock *sk,
                                    struct netlink_broadcast_data *p)
{
        struct netlink_sock *nlk = nlk_sk(sk);
        int val;

        if (p->exclude_sk == sk)
                return;

        if (nlk->portid == p->portid || p->group - 1 >= nlk->ngroups ||
            !test_bit(p->group - 1, nlk->groups))
                return;

        if (!net_eq(sock_net(sk), p->net)) {
                if (!nlk_test_bit(LISTEN_ALL_NSID, sk))
                        return;

                if (!peernet_has_id(sock_net(sk), p->net))
                        return;

                if (!file_ns_capable(sk->sk_socket->file, p->net->user_ns,
                                     CAP_NET_BROADCAST))
                        return;
        }

        if (p->failure) {
                netlink_overrun(sk);
                return;
        }

        sock_hold(sk);
        if (p->skb2 == NULL) {
                if (skb_shared(p->skb)) {
                        p->skb2 = skb_clone(p->skb, p->allocation);
                } else {
                        p->skb2 = skb_get(p->skb);
                        /*
                         * skb ownership may have been set when
                         * delivered to a previous socket.
                         */
                        skb_orphan(p->skb2);
                }
        }
        if (p->skb2 == NULL) {
                netlink_overrun(sk);
                /* Clone failed. Notify ALL listeners. */
                p->failure = 1;
                if (nlk_test_bit(BROADCAST_SEND_ERROR, sk))
                        p->delivery_failure = 1;
                goto out;
        }

        if (p->tx_filter && p->tx_filter(sk, p->skb2, p->tx_data)) {
                kfree_skb(p->skb2);
                p->skb2 = NULL;
                goto out;
        }

        if (sk_filter(sk, p->skb2)) {
                kfree_skb(p->skb2);
                p->skb2 = NULL;
                goto out;
        }

        NETLINK_CB(p->skb2).nsid_is_set = false;
        if (!net_eq(sock_net(sk), p->net)) {
                NETLINK_CB(p->skb2).nsid = peernet2id(sock_net(sk), p->net);
                if (NETLINK_CB(p->skb2).nsid != NETNSA_NSID_NOT_ASSIGNED)
                        NETLINK_CB(p->skb2).nsid_is_set = true;
        }

        val = netlink_broadcast_deliver(sk, p->skb2);
        if (val < 0) {
                netlink_overrun(sk);
                if (nlk_test_bit(BROADCAST_SEND_ERROR, sk))
                        p->delivery_failure = 1;
        } else {
                p->congested |= val;
                p->delivered = 1;
                p->skb2 = NULL;
        }
out:
        sock_put(sk);
}

int netlink_broadcast_filtered(struct sock *ssk, struct sk_buff *skb,
                               u32 portid,
                               u32 group, gfp_t allocation,
                               netlink_filter_fn filter,
                               void *filter_data)
{
        struct net *net = sock_net(ssk);
        struct netlink_broadcast_data info;
        struct sock *sk;

        skb = netlink_trim(skb, allocation);

        info.exclude_sk = ssk;
        info.net = net;
        info.portid = portid;
        info.group = group;
        info.failure = 0;
        info.delivery_failure = 0;
        info.congested = 0;
        info.delivered = 0;
        info.allocation = allocation;
        info.skb = skb;
        info.skb2 = NULL;
        info.tx_filter = filter;
        info.tx_data = filter_data;

        /* While we sleep in clone, do not allow to change socket list */

        netlink_lock_table();

        sk_for_each_bound(sk, &nl_table[ssk->sk_protocol].mc_list)
                do_one_broadcast(sk, &info);

        consume_skb(skb);

        netlink_unlock_table();

        if (info.delivery_failure) {
                kfree_skb(info.skb2);
                return -ENOBUFS;
        }
        consume_skb(info.skb2);

        if (info.delivered) {
                if (info.congested && gfpflags_allow_blocking(allocation))
                        yield();
                return 0;
        }
        return -ESRCH;
}
EXPORT_SYMBOL(netlink_broadcast_filtered);

int netlink_broadcast(struct sock *ssk, struct sk_buff *skb, u32 portid,
                      u32 group, gfp_t allocation)
{
        return netlink_broadcast_filtered(ssk, skb, portid, group, allocation,
                                          NULL, NULL);
}
EXPORT_SYMBOL(netlink_broadcast);

struct netlink_set_err_data {
        struct sock *exclude_sk;
        u32 portid;
        u32 group;
        int code;
};

static int do_one_set_err(struct sock *sk, struct netlink_set_err_data *p)
{
        struct netlink_sock *nlk = nlk_sk(sk);
        int ret = 0;

        if (sk == p->exclude_sk)
                goto out;

        if (!net_eq(sock_net(sk), sock_net(p->exclude_sk)))
                goto out;

        if (nlk->portid == p->portid || p->group - 1 >= nlk->ngroups ||
            !test_bit(p->group - 1, nlk->groups))
                goto out;

        if (p->code == ENOBUFS && nlk_test_bit(RECV_NO_ENOBUFS, sk)) {
                ret = 1;
                goto out;
        }

        WRITE_ONCE(sk->sk_err, p->code);
        sk_error_report(sk);
out:
        return ret;
}

/**
 * netlink_set_err - report error to broadcast listeners
 * @ssk: the kernel netlink socket, as returned by netlink_kernel_create()
 * @portid: the PORTID of a process that we want to skip (if any)
 * @group: the broadcast group that will notice the error
 * @code: error code, must be negative (as usual in kernelspace)
 *
 * This function returns the number of broadcast listeners that have set the
 * NETLINK_NO_ENOBUFS socket option.
 */
int netlink_set_err(struct sock *ssk, u32 portid, u32 group, int code)
{
        struct netlink_set_err_data info;
        unsigned long flags;
        struct sock *sk;
        int ret = 0;

        info.exclude_sk = ssk;
        info.portid = portid;
        info.group = group;
        /* sk->sk_err wants a positive error value */
        info.code = -code;

        read_lock_irqsave(&nl_table_lock, flags);

        sk_for_each_bound(sk, &nl_table[ssk->sk_protocol].mc_list)
                ret += do_one_set_err(sk, &info);

        read_unlock_irqrestore(&nl_table_lock, flags);
        return ret;
}
EXPORT_SYMBOL(netlink_set_err);

/* must be called with netlink table grabbed */
static void netlink_update_socket_mc(struct netlink_sock *nlk,
                                     unsigned int group,
                                     int is_new)
{
        int old, new = !!is_new, subscriptions;

        old = test_bit(group - 1, nlk->groups);
        subscriptions = nlk->subscriptions - old + new;
        __assign_bit(group - 1, nlk->groups, new);
        netlink_update_subscriptions(&nlk->sk, subscriptions);
        netlink_update_listeners(&nlk->sk);
}

static int netlink_setsockopt(struct socket *sock, int level, int optname,
                              sockptr_t optval, unsigned int optlen)
{
        struct sock *sk = sock->sk;
        struct netlink_sock *nlk = nlk_sk(sk);
        unsigned int val = 0;
        int nr = -1;

        if (level != SOL_NETLINK)
                return -ENOPROTOOPT;

        if (optlen >= sizeof(int) &&
            copy_from_sockptr(&val, optval, sizeof(val)))
                return -EFAULT;

        switch (optname) {
        case NETLINK_PKTINFO:
                nr = NETLINK_F_RECV_PKTINFO;
                break;
        case NETLINK_ADD_MEMBERSHIP:
        case NETLINK_DROP_MEMBERSHIP: {
                int err;

                if (!netlink_allowed(sock, NL_CFG_F_NONROOT_RECV))
                        return -EPERM;
                err = netlink_realloc_groups(sk);
                if (err)
                        return err;
                if (!val || val - 1 >= nlk->ngroups)
                        return -EINVAL;
                if (optname == NETLINK_ADD_MEMBERSHIP && nlk->netlink_bind) {
                        err = nlk->netlink_bind(sock_net(sk), val);
                        if (err)
                                return err;
                }
                netlink_table_grab();
                netlink_update_socket_mc(nlk, val,
                                         optname == NETLINK_ADD_MEMBERSHIP);
                netlink_table_ungrab();
                if (optname == NETLINK_DROP_MEMBERSHIP && nlk->netlink_unbind)
                        nlk->netlink_unbind(sock_net(sk), val);

                break;
        }
        case NETLINK_BROADCAST_ERROR:
                nr = NETLINK_F_BROADCAST_SEND_ERROR;
                break;
        case NETLINK_NO_ENOBUFS:
                assign_bit(NETLINK_F_RECV_NO_ENOBUFS, &nlk->flags, val);
                if (val) {
                        clear_bit(NETLINK_S_CONGESTED, &nlk->state);
                        wake_up_interruptible(&nlk->wait);
                }
                break;
        case NETLINK_LISTEN_ALL_NSID:
                if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_BROADCAST))
                        return -EPERM;
                nr = NETLINK_F_LISTEN_ALL_NSID;
                break;
        case NETLINK_CAP_ACK:
                nr = NETLINK_F_CAP_ACK;
                break;
        case NETLINK_EXT_ACK:
                nr = NETLINK_F_EXT_ACK;
                break;
        case NETLINK_GET_STRICT_CHK:
                nr = NETLINK_F_STRICT_CHK;
                break;
        default:
                return -ENOPROTOOPT;
        }
        if (nr >= 0)
                assign_bit(nr, &nlk->flags, val);
        return 0;
}

static int netlink_getsockopt(struct socket *sock, int level, int optname,
                              sockopt_t *opt)
{
        struct sock *sk = sock->sk;
        struct netlink_sock *nlk = nlk_sk(sk);
        unsigned int flag;
        int len, val;
        u32 group;

        if (level != SOL_NETLINK)
                return -ENOPROTOOPT;

        len = opt->optlen;
        if (len < 0)
                return -EINVAL;

        switch (optname) {
        case NETLINK_PKTINFO:
                flag = NETLINK_F_RECV_PKTINFO;
                break;
        case NETLINK_BROADCAST_ERROR:
                flag = NETLINK_F_BROADCAST_SEND_ERROR;
                break;
        case NETLINK_NO_ENOBUFS:
                flag = NETLINK_F_RECV_NO_ENOBUFS;
                break;
        case NETLINK_LIST_MEMBERSHIPS: {
                int pos, idx, shift, err = 0;

                netlink_lock_table();
                for (pos = 0; pos * 8 < nlk->ngroups; pos += sizeof(u32)) {
                        if (len - pos < sizeof(u32))
                                break;

                        idx = pos / sizeof(unsigned long);
                        shift = (pos % sizeof(unsigned long)) * 8;
                        group = (u32)(nlk->groups[idx] >> shift);
                        if (copy_to_iter(&group, sizeof(u32),
                                         &opt->iter_out) != sizeof(u32)) {
                                err = -EFAULT;
                                break;
                        }
                }
                opt->optlen = ALIGN(BITS_TO_BYTES(nlk->ngroups), sizeof(u32));
                netlink_unlock_table();
                return err;
        }
        case NETLINK_LISTEN_ALL_NSID:
                flag = NETLINK_F_LISTEN_ALL_NSID;
                break;
        case NETLINK_CAP_ACK:
                flag = NETLINK_F_CAP_ACK;
                break;
        case NETLINK_EXT_ACK:
                flag = NETLINK_F_EXT_ACK;
                break;
        case NETLINK_GET_STRICT_CHK:
                flag = NETLINK_F_STRICT_CHK;
                break;
        default:
                return -ENOPROTOOPT;
        }

        if (len < sizeof(int))
                return -EINVAL;

        len = sizeof(int);
        val = test_bit(flag, &nlk->flags);

        opt->optlen = len;
        if (copy_to_iter(&val, len, &opt->iter_out) != len)
                return -EFAULT;

        return 0;
}

static void netlink_cmsg_recv_pktinfo(struct msghdr *msg, struct sk_buff *skb)
{
        struct nl_pktinfo info;

        info.group = NETLINK_CB(skb).dst_group;
        put_cmsg(msg, SOL_NETLINK, NETLINK_PKTINFO, sizeof(info), &info);
}

static void netlink_cmsg_listen_all_nsid(struct sock *sk, struct msghdr *msg,
                                         struct sk_buff *skb)
{
        if (!NETLINK_CB(skb).nsid_is_set)
                return;

        put_cmsg(msg, SOL_NETLINK, NETLINK_LISTEN_ALL_NSID, sizeof(int),
                 &NETLINK_CB(skb).nsid);
}

static int netlink_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
{
        struct sock *sk = sock->sk;
        struct netlink_sock *nlk = nlk_sk(sk);
        DECLARE_SOCKADDR(struct sockaddr_nl *, addr, msg->msg_name);
        u32 dst_portid;
        u32 dst_group;
        struct sk_buff *skb;
        int err;
        struct scm_cookie scm;
        u32 netlink_skb_flags = 0;

        if (msg->msg_flags & MSG_OOB)
                return -EOPNOTSUPP;

        if (len == 0) {
                pr_warn_once("Zero length message leads to an empty skb\n");
                return -ENODATA;
        }

        err = scm_send(sock, msg, &scm, true);
        if (err < 0)
                return err;

        if (msg->msg_namelen) {
                err = -EINVAL;
                if (msg->msg_namelen < sizeof(struct sockaddr_nl))
                        goto out;
                if (addr->nl_family != AF_NETLINK)
                        goto out;
                dst_portid = addr->nl_pid;
                dst_group = ffs(addr->nl_groups);
                err =  -EPERM;
                if ((dst_group || dst_portid) &&
                    !netlink_allowed(sock, NL_CFG_F_NONROOT_SEND))
                        goto out;
                netlink_skb_flags |= NETLINK_SKB_DST;
        } else {
                /* Paired with WRITE_ONCE() in netlink_connect() */
                dst_portid = READ_ONCE(nlk->dst_portid);
                dst_group = READ_ONCE(nlk->dst_group);
        }

        /* Paired with WRITE_ONCE() in netlink_insert() */
        if (!READ_ONCE(nlk->bound)) {
                err = netlink_autobind(sock);
                if (err)
                        goto out;
        } else {
                /* Ensure nlk is hashed and visible. */
                smp_rmb();
        }

        err = -EMSGSIZE;
        if (len > sk->sk_sndbuf - 32)
                goto out;
        err = -ENOBUFS;
        skb = netlink_alloc_large_skb(len, dst_group);
        if (skb == NULL)
                goto out;

        NETLINK_CB(skb).portid        = nlk->portid;
        NETLINK_CB(skb).dst_group = dst_group;
        NETLINK_CB(skb).creds        = scm.creds;
        NETLINK_CB(skb).flags        = netlink_skb_flags;

        err = -EFAULT;
        if (memcpy_from_msg(skb_put(skb, len), msg, len)) {
                kfree_skb(skb);
                goto out;
        }

        err = security_netlink_send(sk, skb);
        if (err) {
                kfree_skb(skb);
                goto out;
        }

        if (dst_group) {
                refcount_inc(&skb->users);
                netlink_broadcast(sk, skb, dst_portid, dst_group, GFP_KERNEL);
        }
        err = netlink_unicast(sk, skb, dst_portid, msg->msg_flags & MSG_DONTWAIT);

out:
        scm_destroy(&scm);
        return err;
}

static int netlink_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
                           int flags)
{
        struct scm_cookie scm;
        struct sock *sk = sock->sk;
        struct netlink_sock *nlk = nlk_sk(sk);
        size_t copied, max_recvmsg_len;
        struct sk_buff *skb, *data_skb;
        int err, ret;

        if (flags & MSG_OOB)
                return -EOPNOTSUPP;

        copied = 0;

        skb = skb_recv_datagram(sk, flags, &err);
        if (skb == NULL)
                goto out;

        data_skb = skb;

#ifdef CONFIG_COMPAT_NETLINK_MESSAGES
        if (unlikely(skb_shinfo(skb)->frag_list)) {
                /*
                 * If this skb has a frag_list, then here that means that we
                 * will have to use the frag_list skb's data for compat tasks
                 * and the regular skb's data for normal (non-compat) tasks.
                 *
                 * If we need to send the compat skb, assign it to the
                 * 'data_skb' variable so that it will be used below for data
                 * copying. We keep 'skb' for everything else, including
                 * freeing both later.
                 */
                if (flags & MSG_CMSG_COMPAT)
                        data_skb = skb_shinfo(skb)->frag_list;
        }
#endif

        /* Record the max length of recvmsg() calls for future allocations */
        max_recvmsg_len = max(READ_ONCE(nlk->max_recvmsg_len), len);
        max_recvmsg_len = min_t(size_t, max_recvmsg_len,
                                SKB_WITH_OVERHEAD(32768));
        WRITE_ONCE(nlk->max_recvmsg_len, max_recvmsg_len);

        copied = data_skb->len;
        if (len < copied) {
                msg->msg_flags |= MSG_TRUNC;
                copied = len;
        }

        err = skb_copy_datagram_msg(data_skb, 0, msg, copied);

        if (msg->msg_name) {
                DECLARE_SOCKADDR(struct sockaddr_nl *, addr, msg->msg_name);
                addr->nl_family = AF_NETLINK;
                addr->nl_pad    = 0;
                addr->nl_pid        = NETLINK_CB(skb).portid;
                addr->nl_groups        = netlink_group_mask(NETLINK_CB(skb).dst_group);
                msg->msg_namelen = sizeof(*addr);
        }

        if (nlk_test_bit(RECV_PKTINFO, sk))
                netlink_cmsg_recv_pktinfo(msg, skb);
        if (nlk_test_bit(LISTEN_ALL_NSID, sk))
                netlink_cmsg_listen_all_nsid(sk, msg, skb);

        memset(&scm, 0, sizeof(scm));
        scm.creds = *NETLINK_CREDS(skb);
        if (flags & MSG_TRUNC)
                copied = data_skb->len;

        skb_free_datagram(sk, skb);

        if (READ_ONCE(nlk->cb_running) &&
            atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf / 2) {
                ret = netlink_dump(sk, false);
                if (ret) {
                        WRITE_ONCE(sk->sk_err, -ret);
                        sk_error_report(sk);
                }
        }

        scm_recv(sock, msg, &scm, flags);
out:
        netlink_rcv_wake(sk);
        return err ? : copied;
}

static void netlink_data_ready(struct sock *sk)
{
        BUG();
}

/*
 *        We export these functions to other modules. They provide a
 *        complete set of kernel non-blocking support for message
 *        queueing.
 */

struct sock *
__netlink_kernel_create(struct net *net, int unit, struct module *module,
                        struct netlink_kernel_cfg *cfg)
{
        struct socket *sock;
        struct sock *sk;
        struct netlink_sock *nlk;
        struct listeners *listeners = NULL;
        unsigned int groups;

        BUG_ON(!nl_table);

        if (unit < 0 || unit >= MAX_LINKS)
                return NULL;

        if (sock_create_lite(PF_NETLINK, SOCK_DGRAM, unit, &sock))
                return NULL;

        if (__netlink_create(net, sock, unit, 1) < 0)
                goto out_sock_release_nosk;

        sk = sock->sk;

        if (!cfg || cfg->groups < 32)
                groups = 32;
        else
                groups = cfg->groups;

        listeners = kzalloc(sizeof(*listeners) + NLGRPSZ(groups), GFP_KERNEL);
        if (!listeners)
                goto out_sock_release;

        sk->sk_data_ready = netlink_data_ready;
        if (cfg && cfg->input)
                nlk_sk(sk)->netlink_rcv = cfg->input;

        if (netlink_insert(sk, 0))
                goto out_sock_release;

        nlk = nlk_sk(sk);
        set_bit(NETLINK_F_KERNEL_SOCKET, &nlk->flags);

        netlink_table_grab();
        if (!nl_table[unit].registered) {
                nl_table[unit].groups = groups;
                rcu_assign_pointer(nl_table[unit].listeners, listeners);
                nl_table[unit].module = module;
                if (cfg) {
                        nl_table[unit].bind = cfg->bind;
                        nl_table[unit].unbind = cfg->unbind;
                        nl_table[unit].release = cfg->release;
                        nl_table[unit].flags = cfg->flags;
                }
                nl_table[unit].registered = 1;
        } else {
                kfree(listeners);
                nl_table[unit].registered++;
        }
        netlink_table_ungrab();
        return sk;

out_sock_release:
        kfree(listeners);
        netlink_kernel_release(sk);
        return NULL;

out_sock_release_nosk:
        sock_release(sock);
        return NULL;
}
EXPORT_SYMBOL(__netlink_kernel_create);

void
netlink_kernel_release(struct sock *sk)
{
        if (sk == NULL || sk->sk_socket == NULL)
                return;

        sock_release(sk->sk_socket);
}
EXPORT_SYMBOL(netlink_kernel_release);

int __netlink_change_ngroups(struct sock *sk, unsigned int groups)
{
        struct listeners *new, *old;
        struct netlink_table *tbl = &nl_table[sk->sk_protocol];

        if (groups < 32)
                groups = 32;

        if (NLGRPSZ(tbl->groups) < NLGRPSZ(groups)) {
                new = kzalloc(sizeof(*new) + NLGRPSZ(groups), GFP_ATOMIC);
                if (!new)
                        return -ENOMEM;
                old = nl_deref_protected(tbl->listeners);
                memcpy(new->masks, old->masks, NLGRPSZ(tbl->groups));
                rcu_assign_pointer(tbl->listeners, new);

                kfree_rcu(old, rcu);
        }
        tbl->groups = groups;

        return 0;
}

/**
 * netlink_change_ngroups - change number of multicast groups
 *
 * This changes the number of multicast groups that are available
 * on a certain netlink family. Note that it is not possible to
 * change the number of groups to below 32. Also note that it does
 * not implicitly clear listeners from groups that are removed when
 * the number of groups is reduced.
 *
 * @sk: The kernel netlink socket, as returned by netlink_kernel_create().
 * @groups: The new number of groups.
 */
int netlink_change_ngroups(struct sock *sk, unsigned int groups)
{
        int err;

        netlink_table_grab();
        err = __netlink_change_ngroups(sk, groups);
        netlink_table_ungrab();

        return err;
}

void __netlink_clear_multicast_users(struct sock *ksk, unsigned int group)
{
        struct sock *sk;
        struct netlink_table *tbl = &nl_table[ksk->sk_protocol];
        struct hlist_node *tmp;

        sk_for_each_bound_safe(sk, tmp, &tbl->mc_list)
                netlink_update_socket_mc(nlk_sk(sk), group, 0);
}

struct nlmsghdr *
__nlmsg_put(struct sk_buff *skb, u32 portid, u32 seq, int type, int len, int flags)
{
        struct nlmsghdr *nlh;
        int size = nlmsg_msg_size(len);

        nlh = skb_put(skb, NLMSG_ALIGN(size));
        nlh->nlmsg_type = type;
        nlh->nlmsg_len = size;
        nlh->nlmsg_flags = flags;
        nlh->nlmsg_pid = portid;
        nlh->nlmsg_seq = seq;
        if (!__builtin_constant_p(size) || NLMSG_ALIGN(size) - size != 0)
                memset(nlmsg_data(nlh) + len, 0, NLMSG_ALIGN(size) - size);
        return nlh;
}
EXPORT_SYMBOL(__nlmsg_put);

static size_t
netlink_ack_tlv_len(struct netlink_sock *nlk, int err,
                    const struct netlink_ext_ack *extack)
{
        size_t tlvlen;

        if (!extack || !test_bit(NETLINK_F_EXT_ACK, &nlk->flags))
                return 0;

        tlvlen = 0;
        if (extack->_msg)
                tlvlen += nla_total_size(strlen(extack->_msg) + 1);
        if (extack->cookie_len)
                tlvlen += nla_total_size(extack->cookie_len);

        /* Following attributes are only reported as error (not warning) */
        if (!err)
                return tlvlen;

        if (extack->bad_attr)
                tlvlen += nla_total_size(sizeof(u32));
        if (extack->policy)
                tlvlen += netlink_policy_dump_attr_size_estimate(extack->policy);
        if (extack->miss_type)
                tlvlen += nla_total_size(sizeof(u32));
        if (extack->miss_nest)
                tlvlen += nla_total_size(sizeof(u32));

        return tlvlen;
}

static bool nlmsg_check_in_payload(const struct nlmsghdr *nlh, const void *addr)
{
        return !WARN_ON(addr < nlmsg_data(nlh) ||
                        addr - (const void *) nlh >= nlh->nlmsg_len);
}

static void
netlink_ack_tlv_fill(struct sk_buff *skb, const struct nlmsghdr *nlh, int err,
                     const struct netlink_ext_ack *extack)
{
        if (extack->_msg)
                WARN_ON(nla_put_string(skb, NLMSGERR_ATTR_MSG, extack->_msg));
        if (extack->cookie_len)
                WARN_ON(nla_put(skb, NLMSGERR_ATTR_COOKIE,
                                extack->cookie_len, extack->cookie));

        if (!err)
                return;

        if (extack->bad_attr && nlmsg_check_in_payload(nlh, extack->bad_attr))
                WARN_ON(nla_put_u32(skb, NLMSGERR_ATTR_OFFS,
                                    (u8 *)extack->bad_attr - (const u8 *)nlh));
        if (extack->policy)
                netlink_policy_dump_write_attr(skb, extack->policy,
                                               NLMSGERR_ATTR_POLICY);
        if (extack->miss_type)
                WARN_ON(nla_put_u32(skb, NLMSGERR_ATTR_MISS_TYPE,
                                    extack->miss_type));
        if (extack->miss_nest && nlmsg_check_in_payload(nlh, extack->miss_nest))
                WARN_ON(nla_put_u32(skb, NLMSGERR_ATTR_MISS_NEST,
                                    (u8 *)extack->miss_nest - (const u8 *)nlh));
}

/*
 * It looks a bit ugly.
 * It would be better to create kernel thread.
 */

static int netlink_dump_done(struct netlink_sock *nlk, struct sk_buff *skb,
                             struct netlink_callback *cb,
                             struct netlink_ext_ack *extack)
{
        struct nlmsghdr *nlh;
        size_t extack_len;

        nlh = nlmsg_put_answer(skb, cb, NLMSG_DONE, sizeof(nlk->dump_done_errno),
                               NLM_F_MULTI | cb->answer_flags);
        if (WARN_ON(!nlh))
                return -ENOBUFS;

        nl_dump_check_consistent(cb, nlh);
        memcpy(nlmsg_data(nlh), &nlk->dump_done_errno, sizeof(nlk->dump_done_errno));

        extack_len = netlink_ack_tlv_len(nlk, nlk->dump_done_errno, extack);
        if (extack_len) {
                nlh->nlmsg_flags |= NLM_F_ACK_TLVS;
                if (skb_tailroom(skb) >= extack_len) {
                        netlink_ack_tlv_fill(skb, cb->nlh,
                                             nlk->dump_done_errno, extack);
                        nlmsg_end(skb, nlh);
                }
        }

        return 0;
}

static int netlink_dump(struct sock *sk, bool lock_taken)
{
        struct netlink_sock *nlk = nlk_sk(sk);
        struct netlink_ext_ack extack = {};
        struct netlink_callback *cb;
        struct sk_buff *skb = NULL;
        unsigned int rmem, rcvbuf;
        size_t max_recvmsg_len;
        struct module *module;
        int err = -ENOBUFS;
        int alloc_min_size;
        int alloc_size;

        if (!lock_taken)
                mutex_lock(&nlk->nl_cb_mutex);
        if (!nlk->cb_running) {
                err = -EINVAL;
                goto errout_skb;
        }

        /* NLMSG_GOODSIZE is small to avoid high order allocations being
         * required, but it makes sense to _attempt_ a 32KiB allocation
         * to reduce number of system calls on dump operations, if user
         * ever provided a big enough buffer.
         */
        cb = &nlk->cb;
        alloc_min_size = max_t(int, cb->min_dump_alloc, NLMSG_GOODSIZE);

        max_recvmsg_len = READ_ONCE(nlk->max_recvmsg_len);
        if (alloc_min_size < max_recvmsg_len) {
                alloc_size = max_recvmsg_len;
                skb = alloc_skb(alloc_size,
                                (GFP_KERNEL & ~__GFP_DIRECT_RECLAIM) |
                                __GFP_NOWARN | __GFP_NORETRY);
        }
        if (!skb) {
                alloc_size = alloc_min_size;
                skb = alloc_skb(alloc_size, GFP_KERNEL);
        }
        if (!skb)
                goto errout_skb;

        rcvbuf = READ_ONCE(sk->sk_rcvbuf);
        rmem = atomic_add_return(skb->truesize, &sk->sk_rmem_alloc);
        if (rmem != skb->truesize && rmem >= rcvbuf) {
                atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
                goto errout_skb;
        }

        /* Trim skb to allocated size. User is expected to provide buffer as
         * large as max(min_dump_alloc, 32KiB (max_recvmsg_len capped at
         * netlink_recvmsg())). dump will pack as many smaller messages as
         * could fit within the allocated skb. skb is typically allocated
         * with larger space than required (could be as much as near 2x the
         * requested size with align to next power of 2 approach). Allowing
         * dump to use the excess space makes it difficult for a user to have a
         * reasonable static buffer based on the expected largest dump of a
         * single netdev. The outcome is MSG_TRUNC error.
         */
        skb_reserve(skb, skb_tailroom(skb) - alloc_size);

        /* Make sure malicious BPF programs can not read unitialized memory
         * from skb->head -> skb->data
         */
        skb_reset_network_header(skb);
        skb_reset_mac_header(skb);

        netlink_skb_set_owner_r(skb, sk);

        if (nlk->dump_done_errno > 0) {
                cb->extack = &extack;

                nlk->dump_done_errno = cb->dump(skb, cb);

                /* EMSGSIZE plus something already in the skb means
                 * that there's more to dump but current skb has filled up.
                 * If the callback really wants to return EMSGSIZE to user space
                 * it needs to do so again, on the next cb->dump() call,
                 * without putting data in the skb.
                 */
                if (nlk->dump_done_errno == -EMSGSIZE && skb->len)
                        nlk->dump_done_errno = skb->len;

                cb->extack = NULL;
        }

        if (nlk->dump_done_errno > 0 ||
            skb_tailroom(skb) < nlmsg_total_size(sizeof(nlk->dump_done_errno))) {
                mutex_unlock(&nlk->nl_cb_mutex);

                if (sk_filter(sk, skb))
                        kfree_skb(skb);
                else
                        __netlink_sendskb(sk, skb);
                return 0;
        }

        if (netlink_dump_done(nlk, skb, cb, &extack))
                goto errout_skb;

#ifdef CONFIG_COMPAT_NETLINK_MESSAGES
        /* frag_list skb's data is used for compat tasks
         * and the regular skb's data for normal (non-compat) tasks.
         * See netlink_recvmsg().
         */
        if (unlikely(skb_shinfo(skb)->frag_list)) {
                if (netlink_dump_done(nlk, skb_shinfo(skb)->frag_list, cb, &extack))
                        goto errout_skb;
        }
#endif

        if (sk_filter(sk, skb))
                kfree_skb(skb);
        else
                __netlink_sendskb(sk, skb);

        if (cb->done)
                cb->done(cb);

        WRITE_ONCE(nlk->cb_running, false);
        module = cb->module;
        skb = cb->skb;
        mutex_unlock(&nlk->nl_cb_mutex);
        module_put(module);
        consume_skb(skb);
        return 0;

errout_skb:
        mutex_unlock(&nlk->nl_cb_mutex);
        kfree_skb(skb);
        return err;
}

int __netlink_dump_start(struct sock *ssk, struct sk_buff *skb,
                         const struct nlmsghdr *nlh,
                         struct netlink_dump_control *control)
{
        struct netlink_callback *cb;
        struct netlink_sock *nlk;
        struct sock *sk;
        int ret;

        refcount_inc(&skb->users);

        sk = netlink_lookup(sock_net(ssk), ssk->sk_protocol, NETLINK_CB(skb).portid);
        if (sk == NULL) {
                ret = -ECONNREFUSED;
                goto error_free;
        }

        nlk = nlk_sk(sk);
        mutex_lock(&nlk->nl_cb_mutex);
        /* A dump is in progress... */
        if (nlk->cb_running) {
                ret = -EBUSY;
                goto error_unlock;
        }
        /* add reference of module which cb->dump belongs to */
        if (!try_module_get(control->module)) {
                ret = -EPROTONOSUPPORT;
                goto error_unlock;
        }

        cb = &nlk->cb;
        memset(cb, 0, sizeof(*cb));
        cb->dump = control->dump;
        cb->done = control->done;
        cb->nlh = nlh;
        cb->data = control->data;
        cb->module = control->module;
        cb->min_dump_alloc = control->min_dump_alloc;
        cb->flags = control->flags;
        cb->skb = skb;

        cb->strict_check = nlk_test_bit(STRICT_CHK, NETLINK_CB(skb).sk);

        if (control->start) {
                cb->extack = control->extack;
                ret = control->start(cb);
                cb->extack = NULL;
                if (ret)
                        goto error_put;
        }

        WRITE_ONCE(nlk->cb_running, true);
        nlk->dump_done_errno = INT_MAX;

        ret = netlink_dump(sk, true);

        sock_put(sk);

        if (ret)
                return ret;

        /* We successfully started a dump, by returning -EINTR we
         * signal not to send ACK even if it was requested.
         */
        return -EINTR;

error_put:
        module_put(control->module);
error_unlock:
        sock_put(sk);
        mutex_unlock(&nlk->nl_cb_mutex);
error_free:
        kfree_skb(skb);
        return ret;
}
EXPORT_SYMBOL(__netlink_dump_start);

void netlink_ack(struct sk_buff *in_skb, struct nlmsghdr *nlh, int err,
                 const struct netlink_ext_ack *extack)
{
        struct sk_buff *skb;
        struct nlmsghdr *rep;
        struct nlmsgerr *errmsg;
        size_t payload = sizeof(*errmsg);
        struct netlink_sock *nlk = nlk_sk(NETLINK_CB(in_skb).sk);
        unsigned int flags = 0;
        size_t tlvlen;

        /* Error messages get the original request appended, unless the user
         * requests to cap the error message, and get extra error data if
         * requested.
         */
        if (err && !test_bit(NETLINK_F_CAP_ACK, &nlk->flags))
                payload += nlmsg_len(nlh);
        else
                flags |= NLM_F_CAPPED;

        tlvlen = netlink_ack_tlv_len(nlk, err, extack);
        if (tlvlen)
                flags |= NLM_F_ACK_TLVS;

        skb = nlmsg_new(payload + tlvlen, GFP_KERNEL);
        if (!skb)
                goto err_skb;

        rep = nlmsg_put(skb, NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
                        NLMSG_ERROR, sizeof(*errmsg), flags);
        if (!rep)
                goto err_bad_put;
        errmsg = nlmsg_data(rep);
        errmsg->error = err;
        errmsg->msg = *nlh;

        if (!(flags & NLM_F_CAPPED)) {
                if (!nlmsg_append(skb, nlmsg_len(nlh)))
                        goto err_bad_put;

                memcpy(nlmsg_data(&errmsg->msg), nlmsg_data(nlh),
                       nlmsg_len(nlh));
        }

        if (tlvlen)
                netlink_ack_tlv_fill(skb, nlh, err, extack);

        nlmsg_end(skb, rep);

        nlmsg_unicast(in_skb->sk, skb, NETLINK_CB(in_skb).portid);

        return;

err_bad_put:
        nlmsg_free(skb);
err_skb:
        WRITE_ONCE(NETLINK_CB(in_skb).sk->sk_err, ENOBUFS);
        sk_error_report(NETLINK_CB(in_skb).sk);
}
EXPORT_SYMBOL(netlink_ack);

int netlink_rcv_skb(struct sk_buff *skb, int (*cb)(struct sk_buff *,
                                                   struct nlmsghdr *,
                                                   struct netlink_ext_ack *))
{
        struct netlink_ext_ack extack;
        struct nlmsghdr *nlh;
        int err;

        while (skb->len >= nlmsg_total_size(0)) {
                int msglen;

                memset(&extack, 0, sizeof(extack));
                nlh = nlmsg_hdr(skb);
                err = 0;

                if (nlh->nlmsg_len < NLMSG_HDRLEN || skb->len < nlh->nlmsg_len)
                        return 0;

                /* Only requests are handled by the kernel */
                if (!(nlh->nlmsg_flags & NLM_F_REQUEST))
                        goto ack;

                /* Skip control messages */
                if (nlh->nlmsg_type < NLMSG_MIN_TYPE)
                        goto ack;

                err = cb(skb, nlh, &extack);
                if (err == -EINTR)
                        goto skip;

ack:
                if (nlh->nlmsg_flags & NLM_F_ACK || err)
                        netlink_ack(skb, nlh, err, &extack);

skip:
                msglen = NLMSG_ALIGN(nlh->nlmsg_len);
                if (msglen > skb->len)
                        msglen = skb->len;
                skb_pull(skb, msglen);
        }

        return 0;
}
EXPORT_SYMBOL(netlink_rcv_skb);

/**
 * nlmsg_notify - send a notification netlink message
 * @sk: netlink socket to use
 * @skb: notification message
 * @portid: destination netlink portid for reports or 0
 * @group: destination multicast group or 0
 * @report: 1 to report back, 0 to disable
 * @flags: allocation flags
 */
int nlmsg_notify(struct sock *sk, struct sk_buff *skb, u32 portid,
                 unsigned int group, int report, gfp_t flags)
{
        int err = 0;

        if (group) {
                int exclude_portid = 0;

                if (report) {
                        refcount_inc(&skb->users);
                        exclude_portid = portid;
                }

                /* errors reported via destination sk->sk_err, but propagate
                 * delivery errors if NETLINK_BROADCAST_ERROR flag is set */
                err = nlmsg_multicast(sk, skb, exclude_portid, group, flags);
                if (err == -ESRCH)
                        err = 0;
        }

        if (report) {
                int err2;

                err2 = nlmsg_unicast(sk, skb, portid);
                if (!err)
                        err = err2;
        }

        return err;
}
EXPORT_SYMBOL(nlmsg_notify);

#ifdef CONFIG_PROC_FS
struct nl_seq_iter {
        struct seq_net_private p;
        struct rhashtable_iter hti;
        int link;
};

static void netlink_walk_start(struct nl_seq_iter *iter)
{
        rhashtable_walk_enter(&nl_table[iter->link].hash, &iter->hti);
        rhashtable_walk_start(&iter->hti);
}

static void netlink_walk_stop(struct nl_seq_iter *iter)
{
        rhashtable_walk_stop(&iter->hti);
        rhashtable_walk_exit(&iter->hti);
}

static void *__netlink_seq_next(struct seq_file *seq)
{
        struct nl_seq_iter *iter = seq->private;
        struct netlink_sock *nlk;

        do {
                for (;;) {
                        nlk = rhashtable_walk_next(&iter->hti);

                        if (IS_ERR(nlk)) {
                                if (PTR_ERR(nlk) == -EAGAIN)
                                        continue;

                                return nlk;
                        }

                        if (nlk)
                                break;

                        netlink_walk_stop(iter);
                        if (++iter->link >= MAX_LINKS)
                                return NULL;

                        netlink_walk_start(iter);
                }
        } while (sock_net(&nlk->sk) != seq_file_net(seq));

        return nlk;
}

static void *netlink_seq_start(struct seq_file *seq, loff_t *posp)
        __acquires(RCU)
{
        struct nl_seq_iter *iter = seq->private;
        void *obj = SEQ_START_TOKEN;
        loff_t pos;

        iter->link = 0;

        netlink_walk_start(iter);

        for (pos = *posp; pos && obj && !IS_ERR(obj); pos--)
                obj = __netlink_seq_next(seq);

        return obj;
}

static void *netlink_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        ++*pos;
        return __netlink_seq_next(seq);
}

static void netlink_native_seq_stop(struct seq_file *seq, void *v)
{
        struct nl_seq_iter *iter = seq->private;

        if (iter->link >= MAX_LINKS)
                return;

        netlink_walk_stop(iter);
}


static int netlink_native_seq_show(struct seq_file *seq, void *v)
{
        if (v == SEQ_START_TOKEN) {
                seq_puts(seq,
                         "sk               Eth Pid        Groups   "
                         "Rmem     Wmem     Dump  Locks    Drops    Inode\n");
        } else {
                struct sock *s = v;
                struct netlink_sock *nlk = nlk_sk(s);

                seq_printf(seq, "%pK %-3d %-10u %08x %-8d %-8d %-5d %-8d %-8u %-8llu\n",
                           s,
                           s->sk_protocol,
                           nlk->portid,
                           nlk->groups ? (u32)nlk->groups[0] : 0,
                           sk_rmem_alloc_get(s),
                           sk_wmem_alloc_get(s),
                           READ_ONCE(nlk->cb_running),
                           refcount_read(&s->sk_refcnt),
                           sk_drops_read(s),
                           sock_i_ino(s)
                        );

        }
        return 0;
}

#ifdef CONFIG_BPF_SYSCALL
struct bpf_iter__netlink {
        __bpf_md_ptr(struct bpf_iter_meta *, meta);
        __bpf_md_ptr(struct netlink_sock *, sk);
};

DEFINE_BPF_ITER_FUNC(netlink, struct bpf_iter_meta *meta, struct netlink_sock *sk)

static int netlink_prog_seq_show(struct bpf_prog *prog,
                                  struct bpf_iter_meta *meta,
                                  void *v)
{
        struct bpf_iter__netlink ctx;

        meta->seq_num--;  /* skip SEQ_START_TOKEN */
        ctx.meta = meta;
        ctx.sk = nlk_sk((struct sock *)v);
        return bpf_iter_run_prog(prog, &ctx);
}

static int netlink_seq_show(struct seq_file *seq, void *v)
{
        struct bpf_iter_meta meta;
        struct bpf_prog *prog;

        meta.seq = seq;
        prog = bpf_iter_get_info(&meta, false);
        if (!prog)
                return netlink_native_seq_show(seq, v);

        if (v != SEQ_START_TOKEN)
                return netlink_prog_seq_show(prog, &meta, v);

        return 0;
}

static void netlink_seq_stop(struct seq_file *seq, void *v)
{
        struct bpf_iter_meta meta;
        struct bpf_prog *prog;

        if (!v) {
                meta.seq = seq;
                prog = bpf_iter_get_info(&meta, true);
                if (prog)
                        (void)netlink_prog_seq_show(prog, &meta, v);
        }

        netlink_native_seq_stop(seq, v);
}
#else
static int netlink_seq_show(struct seq_file *seq, void *v)
{
        return netlink_native_seq_show(seq, v);
}

static void netlink_seq_stop(struct seq_file *seq, void *v)
{
        netlink_native_seq_stop(seq, v);
}
#endif

static const struct seq_operations netlink_seq_ops = {
        .start  = netlink_seq_start,
        .next   = netlink_seq_next,
        .stop   = netlink_seq_stop,
        .show   = netlink_seq_show,
};
#endif

int netlink_register_notifier(struct notifier_block *nb)
{
        return blocking_notifier_chain_register(&netlink_chain, nb);
}
EXPORT_SYMBOL(netlink_register_notifier);

int netlink_unregister_notifier(struct notifier_block *nb)
{
        return blocking_notifier_chain_unregister(&netlink_chain, nb);
}
EXPORT_SYMBOL(netlink_unregister_notifier);

static const struct proto_ops netlink_ops = {
        .family =        PF_NETLINK,
        .owner =        THIS_MODULE,
        .release =        netlink_release,
        .bind =                netlink_bind,
        .connect =        netlink_connect,
        .socketpair =        sock_no_socketpair,
        .accept =        sock_no_accept,
        .getname =        netlink_getname,
        .poll =                datagram_poll,
        .ioctl =        netlink_ioctl,
        .listen =        sock_no_listen,
        .shutdown =        sock_no_shutdown,
        .setsockopt =        netlink_setsockopt,
        .getsockopt_iter = netlink_getsockopt,
        .sendmsg =        netlink_sendmsg,
        .recvmsg =        netlink_recvmsg,
        .mmap =                sock_no_mmap,
};

static const struct net_proto_family netlink_family_ops = {
        .family = PF_NETLINK,
        .create = netlink_create,
        .owner        = THIS_MODULE,        /* for consistency 8) */
};

static int __net_init netlink_net_init(struct net *net)
{
#ifdef CONFIG_PROC_FS
        if (!proc_create_net("netlink", 0, net->proc_net, &netlink_seq_ops,
                        sizeof(struct nl_seq_iter)))
                return -ENOMEM;
#endif
        return 0;
}

static void __net_exit netlink_net_exit(struct net *net)
{
#ifdef CONFIG_PROC_FS
        remove_proc_entry("netlink", net->proc_net);
#endif
}

static void __init netlink_add_usersock_entry(void)
{
        struct listeners *listeners;
        int groups = 32;

        listeners = kzalloc(sizeof(*listeners) + NLGRPSZ(groups), GFP_KERNEL);
        if (!listeners)
                panic("netlink_add_usersock_entry: Cannot allocate listeners\n");

        netlink_table_grab();

        nl_table[NETLINK_USERSOCK].groups = groups;
        rcu_assign_pointer(nl_table[NETLINK_USERSOCK].listeners, listeners);
        nl_table[NETLINK_USERSOCK].module = THIS_MODULE;
        nl_table[NETLINK_USERSOCK].registered = 1;
        nl_table[NETLINK_USERSOCK].flags = NL_CFG_F_NONROOT_SEND;

        netlink_table_ungrab();
}

static struct pernet_operations __net_initdata netlink_net_ops = {
        .init = netlink_net_init,
        .exit = netlink_net_exit,
};

static inline u32 netlink_hash(const void *data, u32 len, u32 seed)
{
        const struct netlink_sock *nlk = data;
        struct netlink_compare_arg arg;

        netlink_compare_arg_init(&arg, sock_net(&nlk->sk), nlk->portid);
        return jhash2((u32 *)&arg, netlink_compare_arg_len / sizeof(u32), seed);
}

static const struct rhashtable_params netlink_rhashtable_params = {
        .head_offset = offsetof(struct netlink_sock, node),
        .key_len = netlink_compare_arg_len,
        .obj_hashfn = netlink_hash,
        .obj_cmpfn = netlink_compare,
        .automatic_shrinking = true,
};

#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
BTF_ID_LIST_SINGLE(btf_netlink_sock_id, struct, netlink_sock)

static const struct bpf_iter_seq_info netlink_seq_info = {
        .seq_ops                = &netlink_seq_ops,
        .init_seq_private        = bpf_iter_init_seq_net,
        .fini_seq_private        = bpf_iter_fini_seq_net,
        .seq_priv_size                = sizeof(struct nl_seq_iter),
};

static struct bpf_iter_reg netlink_reg_info = {
        .target                        = "netlink",
        .ctx_arg_info_size        = 1,
        .ctx_arg_info                = {
                { offsetof(struct bpf_iter__netlink, sk),
                  PTR_TO_BTF_ID_OR_NULL },
        },
        .seq_info                = &netlink_seq_info,
};

static int __init bpf_iter_register(void)
{
        netlink_reg_info.ctx_arg_info[0].btf_id = *btf_netlink_sock_id;
        return bpf_iter_reg_target(&netlink_reg_info);
}
#endif

static int __init netlink_proto_init(void)
{
        int i;
        int err = proto_register(&netlink_proto, 0);

        if (err != 0)
                goto out;

#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
        err = bpf_iter_register();
        if (err)
                goto out;
#endif

        BUILD_BUG_ON(sizeof(struct netlink_skb_parms) > sizeof_field(struct sk_buff, cb));

        nl_table = kzalloc_objs(*nl_table, MAX_LINKS);
        if (!nl_table)
                goto panic;

        for (i = 0; i < MAX_LINKS; i++) {
                if (rhashtable_init(&nl_table[i].hash,
                                    &netlink_rhashtable_params) < 0)
                        goto panic;
        }

        netlink_add_usersock_entry();

        sock_register(&netlink_family_ops);
        register_pernet_subsys(&netlink_net_ops);
        register_pernet_subsys(&netlink_tap_net_ops);
        /* The netlink device handler may be needed early. */
        rtnetlink_init();
out:
        return err;
panic:
        panic("netlink_init: Cannot allocate nl_table\n");
}

core_initcall(netlink_proto_init);




























































































































































































































































































































































































































































































































    3 






    3 




    3 





    3 







    3 






    3 


































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    4 



    4 



    4 


























    4 





    4 































    4 


    4 











    4 























































































































































































    5 

    5 















    5 




    5 
    5 

    5 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
/*
 * Atheros CARL9170 driver
 *
 * mac80211 interaction code
 *
 * Copyright 2008, Johannes Berg <johannes@sipsolutions.net>
 * Copyright 2009, 2010, Christian Lamparter <chunkeey@googlemail.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; see the file COPYING.  If not, see
 * http://www.gnu.org/licenses/.
 *
 * This file incorporates work covered by the following copyright and
 * permission notice:
 *    Copyright (c) 2007-2008 Atheros Communications, Inc.
 *
 *    Permission to use, copy, modify, and/or distribute this software for any
 *    purpose with or without fee is hereby granted, provided that the above
 *    copyright notice and this permission notice appear in all copies.
 *
 *    THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 *    WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 *    MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 *    ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 *    WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 *    ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 *    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include <linux/slab.h>
#include <linux/module.h>
#include <linux/etherdevice.h>
#include <linux/random.h>
#include <net/mac80211.h>
#include <net/cfg80211.h>
#include "hw.h"
#include "carl9170.h"
#include "cmd.h"

static bool modparam_nohwcrypt;
module_param_named(nohwcrypt, modparam_nohwcrypt, bool, 0444);
MODULE_PARM_DESC(nohwcrypt, "Disable hardware crypto offload.");

int modparam_noht;
module_param_named(noht, modparam_noht, int, 0444);
MODULE_PARM_DESC(noht, "Disable MPDU aggregation.");

#define RATE(_bitrate, _hw_rate, _txpidx, _flags) {        \
        .bitrate        = (_bitrate),                        \
        .flags                = (_flags),                        \
        .hw_value        = (_hw_rate) | (_txpidx) << 4,        \
}

struct ieee80211_rate __carl9170_ratetable[] = {
        RATE(10, 0, 0, 0),
        RATE(20, 1, 1, IEEE80211_RATE_SHORT_PREAMBLE),
        RATE(55, 2, 2, IEEE80211_RATE_SHORT_PREAMBLE),
        RATE(110, 3, 3, IEEE80211_RATE_SHORT_PREAMBLE),
        RATE(60, 0xb, 0, 0),
        RATE(90, 0xf, 0, 0),
        RATE(120, 0xa, 0, 0),
        RATE(180, 0xe, 0, 0),
        RATE(240, 0x9, 0, 0),
        RATE(360, 0xd, 1, 0),
        RATE(480, 0x8, 2, 0),
        RATE(540, 0xc, 3, 0),
};
#undef RATE

#define carl9170_g_ratetable        (__carl9170_ratetable + 0)
#define carl9170_g_ratetable_size        12
#define carl9170_a_ratetable        (__carl9170_ratetable + 4)
#define carl9170_a_ratetable_size        8

/*
 * NB: The hw_value is used as an index into the carl9170_phy_freq_params
 *     array in phy.c so that we don't have to do frequency lookups!
 */
#define CHAN(_freq, _idx) {                \
        .center_freq        = (_freq),        \
        .hw_value        = (_idx),        \
        .max_power        = 18, /* XXX */        \
}

static struct ieee80211_channel carl9170_2ghz_chantable[] = {
        CHAN(2412,  0),
        CHAN(2417,  1),
        CHAN(2422,  2),
        CHAN(2427,  3),
        CHAN(2432,  4),
        CHAN(2437,  5),
        CHAN(2442,  6),
        CHAN(2447,  7),
        CHAN(2452,  8),
        CHAN(2457,  9),
        CHAN(2462, 10),
        CHAN(2467, 11),
        CHAN(2472, 12),
        CHAN(2484, 13),
};

static struct ieee80211_channel carl9170_5ghz_chantable[] = {
        CHAN(4920, 14),
        CHAN(4940, 15),
        CHAN(4960, 16),
        CHAN(4980, 17),
        CHAN(5040, 18),
        CHAN(5060, 19),
        CHAN(5080, 20),
        CHAN(5180, 21),
        CHAN(5200, 22),
        CHAN(5220, 23),
        CHAN(5240, 24),
        CHAN(5260, 25),
        CHAN(5280, 26),
        CHAN(5300, 27),
        CHAN(5320, 28),
        CHAN(5500, 29),
        CHAN(5520, 30),
        CHAN(5540, 31),
        CHAN(5560, 32),
        CHAN(5580, 33),
        CHAN(5600, 34),
        CHAN(5620, 35),
        CHAN(5640, 36),
        CHAN(5660, 37),
        CHAN(5680, 38),
        CHAN(5700, 39),
        CHAN(5745, 40),
        CHAN(5765, 41),
        CHAN(5785, 42),
        CHAN(5805, 43),
        CHAN(5825, 44),
        CHAN(5170, 45),
        CHAN(5190, 46),
        CHAN(5210, 47),
        CHAN(5230, 48),
};
#undef CHAN

#define CARL9170_HT_CAP                                                        \
{                                                                        \
        .ht_supported        = true,                                                \
        .cap                = IEEE80211_HT_CAP_MAX_AMSDU |                        \
                          IEEE80211_HT_CAP_SUP_WIDTH_20_40 |                \
                          IEEE80211_HT_CAP_SGI_40 |                        \
                          IEEE80211_HT_CAP_DSSSCCK40 |                        \
                          IEEE80211_HT_CAP_SM_PS,                        \
        .ampdu_factor        = IEEE80211_HT_MAX_AMPDU_64K,                        \
        .ampdu_density        = IEEE80211_HT_MPDU_DENSITY_8,                        \
        .mcs                = {                                                \
                .rx_mask = { 0xff, 0xff, 0, 0, 0x1, 0, 0, 0, 0, 0, },        \
                .rx_highest = cpu_to_le16(300),                                \
                .tx_params = IEEE80211_HT_MCS_TX_DEFINED,                \
        },                                                                \
}

static struct ieee80211_supported_band carl9170_band_2GHz = {
        .channels        = carl9170_2ghz_chantable,
        .n_channels        = ARRAY_SIZE(carl9170_2ghz_chantable),
        .bitrates        = carl9170_g_ratetable,
        .n_bitrates        = carl9170_g_ratetable_size,
        .ht_cap                = CARL9170_HT_CAP,
};

static struct ieee80211_supported_band carl9170_band_5GHz = {
        .channels        = carl9170_5ghz_chantable,
        .n_channels        = ARRAY_SIZE(carl9170_5ghz_chantable),
        .bitrates        = carl9170_a_ratetable,
        .n_bitrates        = carl9170_a_ratetable_size,
        .ht_cap                = CARL9170_HT_CAP,
};

static void carl9170_ampdu_gc(struct ar9170 *ar)
{
        struct carl9170_sta_tid *tid_info;
        LIST_HEAD(tid_gc);

        rcu_read_lock();
        list_for_each_entry_rcu(tid_info, &ar->tx_ampdu_list, list) {
                spin_lock_bh(&ar->tx_ampdu_list_lock);
                if (tid_info->state == CARL9170_TID_STATE_SHUTDOWN) {
                        tid_info->state = CARL9170_TID_STATE_KILLED;
                        list_del_rcu(&tid_info->list);
                        ar->tx_ampdu_list_len--;
                        list_add_tail(&tid_info->tmp_list, &tid_gc);
                }
                spin_unlock_bh(&ar->tx_ampdu_list_lock);

        }
        rcu_assign_pointer(ar->tx_ampdu_iter, tid_info);
        rcu_read_unlock();

        synchronize_rcu();

        while (!list_empty(&tid_gc)) {
                struct sk_buff *skb;
                tid_info = list_first_entry(&tid_gc, struct carl9170_sta_tid,
                                            tmp_list);

                while ((skb = __skb_dequeue(&tid_info->queue)))
                        carl9170_tx_status(ar, skb, false);

                list_del_init(&tid_info->tmp_list);
                kfree(tid_info);
        }
}

static void carl9170_flush(struct ar9170 *ar, bool drop_queued)
{
        if (drop_queued) {
                int i;

                /*
                 * We can only drop frames which have not been uploaded
                 * to the device yet.
                 */

                for (i = 0; i < ar->hw->queues; i++) {
                        struct sk_buff *skb;

                        while ((skb = skb_dequeue(&ar->tx_pending[i]))) {
                                struct ieee80211_tx_info *info;

                                info = IEEE80211_SKB_CB(skb);
                                if (info->flags & IEEE80211_TX_CTL_AMPDU)
                                        atomic_dec(&ar->tx_ampdu_upload);

                                carl9170_tx_status(ar, skb, false);
                        }
                }
        }

        /* Wait for all other outstanding frames to timeout. */
        if (atomic_read(&ar->tx_total_queued))
                WARN_ON(wait_for_completion_timeout(&ar->tx_flush, HZ) == 0);
}

static void carl9170_flush_ba(struct ar9170 *ar)
{
        struct sk_buff_head free;
        struct carl9170_sta_tid *tid_info;
        struct sk_buff *skb;

        __skb_queue_head_init(&free);

        rcu_read_lock();
        spin_lock_bh(&ar->tx_ampdu_list_lock);
        list_for_each_entry_rcu(tid_info, &ar->tx_ampdu_list, list) {
                if (tid_info->state > CARL9170_TID_STATE_SUSPEND) {
                        tid_info->state = CARL9170_TID_STATE_SUSPEND;

                        spin_lock(&tid_info->lock);
                        while ((skb = __skb_dequeue(&tid_info->queue)))
                                __skb_queue_tail(&free, skb);
                        spin_unlock(&tid_info->lock);
                }
        }
        spin_unlock_bh(&ar->tx_ampdu_list_lock);
        rcu_read_unlock();

        while ((skb = __skb_dequeue(&free)))
                carl9170_tx_status(ar, skb, false);
}

static void carl9170_zap_queues(struct ar9170 *ar)
{
        struct carl9170_vif_info *cvif;
        unsigned int i;

        carl9170_ampdu_gc(ar);

        carl9170_flush_ba(ar);
        carl9170_flush(ar, true);

        for (i = 0; i < ar->hw->queues; i++) {
                spin_lock_bh(&ar->tx_status[i].lock);
                while (!skb_queue_empty(&ar->tx_status[i])) {
                        struct sk_buff *skb;

                        skb = skb_peek(&ar->tx_status[i]);
                        carl9170_tx_get_skb(skb);
                        spin_unlock_bh(&ar->tx_status[i].lock);
                        carl9170_tx_drop(ar, skb);
                        spin_lock_bh(&ar->tx_status[i].lock);
                        carl9170_tx_put_skb(skb);
                }
                spin_unlock_bh(&ar->tx_status[i].lock);
        }

        BUILD_BUG_ON(CARL9170_NUM_TX_LIMIT_SOFT < 1);
        BUILD_BUG_ON(CARL9170_NUM_TX_LIMIT_HARD < CARL9170_NUM_TX_LIMIT_SOFT);
        BUILD_BUG_ON(CARL9170_NUM_TX_LIMIT_HARD >= CARL9170_BAW_BITS);

        /* reinitialize queues statistics */
        memset(&ar->tx_stats, 0, sizeof(ar->tx_stats));
        for (i = 0; i < ar->hw->queues; i++)
                ar->tx_stats[i].limit = CARL9170_NUM_TX_LIMIT_HARD;

        bitmap_zero(ar->mem_bitmap, ar->fw.mem_blocks);

        rcu_read_lock();
        list_for_each_entry_rcu(cvif, &ar->vif_list, list) {
                spin_lock_bh(&ar->beacon_lock);
                dev_kfree_skb_any(cvif->beacon);
                cvif->beacon = NULL;
                spin_unlock_bh(&ar->beacon_lock);
        }
        rcu_read_unlock();

        atomic_set(&ar->tx_ampdu_upload, 0);
        atomic_set(&ar->tx_ampdu_scheduler, 0);
        atomic_set(&ar->tx_total_pending, 0);
        atomic_set(&ar->tx_total_queued, 0);
        atomic_set(&ar->mem_free_blocks, ar->fw.mem_blocks);
}

#define CARL9170_FILL_QUEUE(queue, ai_fs, cwmin, cwmax, _txop)                \
do {                                                                        \
        queue.aifs = ai_fs;                                                \
        queue.cw_min = cwmin;                                                \
        queue.cw_max = cwmax;                                                \
        queue.txop = _txop;                                                \
} while (0)

static int carl9170_op_start(struct ieee80211_hw *hw)
{
        struct ar9170 *ar = hw->priv;
        int err, i;

        mutex_lock(&ar->mutex);

        carl9170_zap_queues(ar);

        /* reset QoS defaults */
        CARL9170_FILL_QUEUE(ar->edcf[AR9170_TXQ_VO], 2, 3,     7, 47);
        CARL9170_FILL_QUEUE(ar->edcf[AR9170_TXQ_VI], 2, 7,    15, 94);
        CARL9170_FILL_QUEUE(ar->edcf[AR9170_TXQ_BE], 3, 15, 1023,  0);
        CARL9170_FILL_QUEUE(ar->edcf[AR9170_TXQ_BK], 7, 15, 1023,  0);
        CARL9170_FILL_QUEUE(ar->edcf[AR9170_TXQ_SPECIAL], 2, 3, 7, 0);

        ar->current_factor = ar->current_density = -1;
        /* "The first key is unique." */
        ar->usedkeys = 1;
        ar->filter_state = 0;
        ar->ps.last_action = jiffies;
        ar->ps.last_slept = jiffies;
        ar->erp_mode = CARL9170_ERP_AUTO;

        /* Set "disable hw crypto offload" whenever the module parameter
         * nohwcrypt is true or if the firmware does not support it.
         */
        ar->disable_offload = modparam_nohwcrypt |
                ar->fw.disable_offload_fw;
        ar->rx_software_decryption = ar->disable_offload;

        for (i = 0; i < ar->hw->queues; i++) {
                ar->queue_stop_timeout[i] = jiffies;
                ar->max_queue_stop_timeout[i] = 0;
        }

        atomic_set(&ar->mem_allocs, 0);

        err = carl9170_usb_open(ar);
        if (err)
                goto out;

        err = carl9170_init_mac(ar);
        if (err)
                goto out;

        err = carl9170_set_qos(ar);
        if (err)
                goto out;

        if (ar->fw.rx_filter) {
                err = carl9170_rx_filter(ar, CARL9170_RX_FILTER_OTHER_RA |
                        CARL9170_RX_FILTER_CTL_OTHER | CARL9170_RX_FILTER_BAD);
                if (err)
                        goto out;
        }

        err = carl9170_write_reg(ar, AR9170_MAC_REG_DMA_TRIGGER,
                                 AR9170_DMA_TRIGGER_RXQ);
        if (err)
                goto out;

        /* Clear key-cache */
        for (i = 0; i < AR9170_CAM_MAX_USER + 4; i++) {
                err = carl9170_upload_key(ar, i, NULL, AR9170_ENC_ALG_NONE,
                                          0, NULL, 0);
                if (err)
                        goto out;

                err = carl9170_upload_key(ar, i, NULL, AR9170_ENC_ALG_NONE,
                                          1, NULL, 0);
                if (err)
                        goto out;

                if (i < AR9170_CAM_MAX_USER) {
                        err = carl9170_disable_key(ar, i);
                        if (err)
                                goto out;
                }
        }

        carl9170_set_state_when(ar, CARL9170_IDLE, CARL9170_STARTED);

        ieee80211_queue_delayed_work(ar->hw, &ar->stat_work,
                round_jiffies(msecs_to_jiffies(CARL9170_STAT_WORK)));

        ieee80211_wake_queues(ar->hw);
        err = 0;

out:
        mutex_unlock(&ar->mutex);
        return err;
}

static void carl9170_cancel_worker(struct ar9170 *ar)
{
        cancel_delayed_work_sync(&ar->stat_work);
        cancel_delayed_work_sync(&ar->tx_janitor);
#ifdef CONFIG_CARL9170_LEDS
        cancel_delayed_work_sync(&ar->led_work);
#endif /* CONFIG_CARL9170_LEDS */
        cancel_work_sync(&ar->ps_work);
        cancel_work_sync(&ar->ping_work);
        cancel_work_sync(&ar->ampdu_work);
}

static void carl9170_op_stop(struct ieee80211_hw *hw, bool suspend)
{
        struct ar9170 *ar = hw->priv;

        carl9170_set_state_when(ar, CARL9170_STARTED, CARL9170_IDLE);

        ieee80211_stop_queues(ar->hw);

        mutex_lock(&ar->mutex);
        if (IS_ACCEPTING_CMD(ar)) {
                RCU_INIT_POINTER(ar->beacon_iter, NULL);

                carl9170_led_set_state(ar, 0);

                /* stop DMA */
                carl9170_write_reg(ar, AR9170_MAC_REG_DMA_TRIGGER, 0);
                carl9170_usb_stop(ar);
        }

        carl9170_zap_queues(ar);
        mutex_unlock(&ar->mutex);

        carl9170_cancel_worker(ar);
}

static void carl9170_restart_work(struct work_struct *work)
{
        struct ar9170 *ar = container_of(work, struct ar9170,
                                         restart_work);
        int err = -EIO;

        ar->usedkeys = 0;
        ar->filter_state = 0;
        carl9170_cancel_worker(ar);

        mutex_lock(&ar->mutex);
        if (!ar->force_usb_reset) {
                err = carl9170_usb_restart(ar);
                if (net_ratelimit()) {
                        if (err)
                                dev_err(&ar->udev->dev, "Failed to restart device (%d).\n", err);
                        else
                                dev_info(&ar->udev->dev, "device restarted successfully.\n");
                }
        }
        carl9170_zap_queues(ar);
        mutex_unlock(&ar->mutex);

        if (!err && !ar->force_usb_reset) {
                ar->restart_counter++;
                atomic_set(&ar->pending_restarts, 0);

                ieee80211_restart_hw(ar->hw);
        } else {
                /*
                 * The reset was unsuccessful and the device seems to
                 * be dead. But there's still one option: a low-level
                 * usb subsystem reset...
                 */

                carl9170_usb_reset(ar);
        }
}

void carl9170_restart(struct ar9170 *ar, const enum carl9170_restart_reasons r)
{
        carl9170_set_state_when(ar, CARL9170_STARTED, CARL9170_IDLE);

        /*
         * Sometimes, an error can trigger several different reset events.
         * By ignoring these *surplus* reset events, the device won't be
         * killed again, right after it has recovered.
         */
        if (atomic_inc_return(&ar->pending_restarts) > 1) {
                dev_dbg(&ar->udev->dev, "ignoring restart (%d)\n", r);
                return;
        }

        ieee80211_stop_queues(ar->hw);

        dev_err(&ar->udev->dev, "restart device (%d)\n", r);

        if (!WARN_ON(r == CARL9170_RR_NO_REASON) ||
            !WARN_ON(r >= __CARL9170_RR_LAST))
                ar->last_reason = r;

        if (!ar->registered)
                return;

        if (!IS_ACCEPTING_CMD(ar) || ar->needs_full_reset)
                ar->force_usb_reset = true;

        ieee80211_queue_work(ar->hw, &ar->restart_work);

        /*
         * At this point, the device instance might have vanished/disabled.
         * So, don't put any code which access the ar9170 struct
         * without proper protection.
         */
}

static void carl9170_ping_work(struct work_struct *work)
{
        struct ar9170 *ar = container_of(work, struct ar9170, ping_work);
        int err;

        if (!IS_STARTED(ar))
                return;

        mutex_lock(&ar->mutex);
        err = carl9170_echo_test(ar, 0xdeadbeef);
        if (err)
                carl9170_restart(ar, CARL9170_RR_UNRESPONSIVE_DEVICE);
        mutex_unlock(&ar->mutex);
}

static int carl9170_init_interface(struct ar9170 *ar,
                                   struct ieee80211_vif *vif)
{
        struct ath_common *common = &ar->common;
        int err;

        if (!vif) {
                WARN_ON_ONCE(IS_STARTED(ar));
                return 0;
        }

        memcpy(common->macaddr, vif->addr, ETH_ALEN);

        /* We have to fall back to software crypto, whenever
         * the user choose to participates in an IBSS. HW
         * offload for IBSS RSN is not supported by this driver.
         *
         * NOTE: If the previous main interface has already
         * disabled hw crypto offload, we have to keep this
         * previous disable_offload setting as it was.
         * Altough ideally, we should notify mac80211 and tell
         * it to forget about any HW crypto offload for now.
         */
        ar->disable_offload |= ((vif->type != NL80211_IFTYPE_STATION) &&
            (vif->type != NL80211_IFTYPE_AP));

        /* The driver used to have P2P GO+CLIENT support,
         * but since this was dropped and we don't know if
         * there are any gremlins lurking in the shadows,
         * so best we keep HW offload disabled for P2P.
         */
        ar->disable_offload |= vif->p2p;

        ar->rx_software_decryption = ar->disable_offload;

        err = carl9170_set_operating_mode(ar);
        return err;
}

static int carl9170_op_add_interface(struct ieee80211_hw *hw,
                                     struct ieee80211_vif *vif)
{
        struct carl9170_vif_info *vif_priv = (void *) vif->drv_priv;
        struct ieee80211_vif *main_vif, *old_main = NULL;
        struct ar9170 *ar = hw->priv;
        int vif_id = -1, err = 0;

        mutex_lock(&ar->mutex);
        rcu_read_lock();
        if (vif_priv->active) {
                /*
                 * Skip the interface structure initialization,
                 * if the vif survived the _restart call.
                 */
                vif_id = vif_priv->id;
                vif_priv->enable_beacon = false;

                spin_lock_bh(&ar->beacon_lock);
                dev_kfree_skb_any(vif_priv->beacon);
                vif_priv->beacon = NULL;
                spin_unlock_bh(&ar->beacon_lock);

                goto init;
        }

        /* Because the AR9170 HW's MAC doesn't provide full support for
         * multiple, independent interfaces [of different operation modes].
         * We have to select ONE main interface [main mode of HW], but we
         * can have multiple slaves [AKA: entry in the ACK-table].
         *
         * The first (from HEAD/TOP) interface in the ar->vif_list is
         * always the main intf. All following intfs in this list
         * are considered to be slave intfs.
         */
        main_vif = carl9170_get_main_vif(ar);

        if (main_vif) {
                switch (main_vif->type) {
                case NL80211_IFTYPE_STATION:
                        if (vif->type == NL80211_IFTYPE_STATION)
                                break;

                        err = -EBUSY;
                        rcu_read_unlock();

                        goto unlock;

                case NL80211_IFTYPE_MESH_POINT:
                case NL80211_IFTYPE_AP:
                        if ((vif->type == NL80211_IFTYPE_STATION) ||
                            (vif->type == NL80211_IFTYPE_AP) ||
                            (vif->type == NL80211_IFTYPE_MESH_POINT))
                                break;

                        err = -EBUSY;
                        rcu_read_unlock();
                        goto unlock;

                default:
                        rcu_read_unlock();
                        goto unlock;
                }
        }

        vif_id = bitmap_find_free_region(&ar->vif_bitmap, ar->fw.vif_num, 0);

        if (vif_id < 0) {
                rcu_read_unlock();

                err = -ENOSPC;
                goto unlock;
        }

        BUG_ON(ar->vif_priv[vif_id].id != vif_id);

        vif_priv->active = true;
        vif_priv->id = vif_id;
        vif_priv->enable_beacon = false;
        ar->vifs++;
        if (old_main) {
                /* We end up in here, if the main interface is being replaced.
                 * Put the new main interface at the HEAD of the list and the
                 * previous inteface will automatically become second in line.
                 */
                list_add_rcu(&vif_priv->list, &ar->vif_list);
        } else {
                /* Add new inteface. If the list is empty, it will become the
                 * main inteface, otherwise it will be slave.
                 */
                list_add_tail_rcu(&vif_priv->list, &ar->vif_list);
        }
        rcu_assign_pointer(ar->vif_priv[vif_id].vif, vif);

init:
        main_vif = carl9170_get_main_vif(ar);

        if (main_vif == vif) {
                rcu_assign_pointer(ar->beacon_iter, vif_priv);
                rcu_read_unlock();

                if (old_main) {
                        struct carl9170_vif_info *old_main_priv =
                                (void *) old_main->drv_priv;
                        /* downgrade old main intf to slave intf.
                         * NOTE: We are no longer under rcu_read_lock.
                         * But we are still holding ar->mutex, so the
                         * vif data [id, addr] is safe.
                         */
                        err = carl9170_mod_virtual_mac(ar, old_main_priv->id,
                                                       old_main->addr);
                        if (err)
                                goto unlock;
                }

                err = carl9170_init_interface(ar, vif);
                if (err)
                        goto unlock;
        } else {
                rcu_read_unlock();
                err = carl9170_mod_virtual_mac(ar, vif_id, vif->addr);

                if (err)
                        goto unlock;
        }

        if (ar->fw.tx_seq_table) {
                err = carl9170_write_reg(ar, ar->fw.tx_seq_table + vif_id * 4,
                                         0);
                if (err)
                        goto unlock;
        }

unlock:
        if (err && (vif_id >= 0)) {
                vif_priv->active = false;
                bitmap_release_region(&ar->vif_bitmap, vif_id, 0);
                ar->vifs--;
                RCU_INIT_POINTER(ar->vif_priv[vif_id].vif, NULL);
                list_del_rcu(&vif_priv->list);
                mutex_unlock(&ar->mutex);
                synchronize_rcu();
        } else {
                if (ar->vifs > 1)
                        ar->ps.off_override |= PS_OFF_VIF;

                mutex_unlock(&ar->mutex);
        }

        return err;
}

static void carl9170_op_remove_interface(struct ieee80211_hw *hw,
                                         struct ieee80211_vif *vif)
{
        struct carl9170_vif_info *vif_priv = (void *) vif->drv_priv;
        struct ieee80211_vif *main_vif;
        struct ar9170 *ar = hw->priv;
        unsigned int id;

        mutex_lock(&ar->mutex);

        if (WARN_ON_ONCE(!vif_priv->active))
                goto unlock;

        ar->vifs--;

        rcu_read_lock();
        main_vif = carl9170_get_main_vif(ar);

        id = vif_priv->id;

        vif_priv->active = false;
        WARN_ON(vif_priv->enable_beacon);
        vif_priv->enable_beacon = false;
        list_del_rcu(&vif_priv->list);
        RCU_INIT_POINTER(ar->vif_priv[id].vif, NULL);

        if (vif == main_vif) {
                rcu_read_unlock();

                if (ar->vifs) {
                        WARN_ON(carl9170_init_interface(ar,
                                        carl9170_get_main_vif(ar)));
                } else {
                        carl9170_set_operating_mode(ar);
                }
        } else {
                rcu_read_unlock();

                WARN_ON(carl9170_mod_virtual_mac(ar, id, NULL));
        }

        carl9170_update_beacon(ar, false);
        carl9170_flush_cab(ar, id);

        spin_lock_bh(&ar->beacon_lock);
        dev_kfree_skb_any(vif_priv->beacon);
        vif_priv->beacon = NULL;
        spin_unlock_bh(&ar->beacon_lock);

        bitmap_release_region(&ar->vif_bitmap, id, 0);

        carl9170_set_beacon_timers(ar);

        if (ar->vifs == 1)
                ar->ps.off_override &= ~PS_OFF_VIF;

unlock:
        mutex_unlock(&ar->mutex);

        synchronize_rcu();
}

void carl9170_ps_check(struct ar9170 *ar)
{
        ieee80211_queue_work(ar->hw, &ar->ps_work);
}

/* caller must hold ar->mutex */
static int carl9170_ps_update(struct ar9170 *ar)
{
        bool ps = false;
        int err = 0;

        if (!ar->ps.off_override)
                ps = (ar->hw->conf.flags & IEEE80211_CONF_PS);

        if (ps != ar->ps.state) {
                err = carl9170_powersave(ar, ps);
                if (err)
                        return err;

                if (ar->ps.state && !ps) {
                        ar->ps.sleep_ms = jiffies_to_msecs(jiffies -
                                ar->ps.last_action);
                }

                if (ps)
                        ar->ps.last_slept = jiffies;

                ar->ps.last_action = jiffies;
                ar->ps.state = ps;
        }

        return 0;
}

static void carl9170_ps_work(struct work_struct *work)
{
        struct ar9170 *ar = container_of(work, struct ar9170,
                                         ps_work);
        mutex_lock(&ar->mutex);
        if (IS_STARTED(ar))
                WARN_ON_ONCE(carl9170_ps_update(ar) != 0);
        mutex_unlock(&ar->mutex);
}

static int carl9170_update_survey(struct ar9170 *ar, bool flush, bool noise)
{
        int err;

        if (noise) {
                err = carl9170_get_noisefloor(ar);
                if (err)
                        return err;
        }

        if (ar->fw.hw_counters) {
                err = carl9170_collect_tally(ar);
                if (err)
                        return err;
        }

        if (flush)
                memset(&ar->tally, 0, sizeof(ar->tally));

        return 0;
}

static void carl9170_stat_work(struct work_struct *work)
{
        struct ar9170 *ar = container_of(work, struct ar9170, stat_work.work);
        int err;

        mutex_lock(&ar->mutex);
        err = carl9170_update_survey(ar, false, true);
        mutex_unlock(&ar->mutex);

        if (err)
                return;

        ieee80211_queue_delayed_work(ar->hw, &ar->stat_work,
                round_jiffies(msecs_to_jiffies(CARL9170_STAT_WORK)));
}

static int carl9170_op_config(struct ieee80211_hw *hw, int radio_idx, u32 changed)
{
        struct ar9170 *ar = hw->priv;
        int err = 0;

        mutex_lock(&ar->mutex);
        if (changed & IEEE80211_CONF_CHANGE_LISTEN_INTERVAL) {
                /* TODO */
                err = 0;
        }

        if (changed & IEEE80211_CONF_CHANGE_PS) {
                err = carl9170_ps_update(ar);
                if (err)
                        goto out;
        }

        if (changed & IEEE80211_CONF_CHANGE_SMPS) {
                /*
                 * We advertise SM_PS disabled (all chains active).
                 * mac80211 may still request mode changes, which we
                 * accept but only support OFF (both chains active).
                 * Static/dynamic SMPS would require firmware support
                 * for chain control that the AR9170 does not provide.
                 */
                err = 0;
        }

        if (changed & IEEE80211_CONF_CHANGE_CHANNEL) {
                enum nl80211_channel_type channel_type =
                        cfg80211_get_chandef_type(&hw->conf.chandef);

                /* adjust slot time for 5 GHz */
                err = carl9170_set_slot_time(ar);
                if (err)
                        goto out;

                err = carl9170_update_survey(ar, true, false);
                if (err)
                        goto out;

                err = carl9170_set_channel(ar, hw->conf.chandef.chan,
                                           channel_type);
                if (err)
                        goto out;

                err = carl9170_update_survey(ar, false, true);
                if (err)
                        goto out;

                err = carl9170_set_dyn_sifs_ack(ar);
                if (err)
                        goto out;

                err = carl9170_set_rts_cts_rate(ar);
                if (err)
                        goto out;
        }

        if (changed & IEEE80211_CONF_CHANGE_POWER) {
                err = carl9170_set_mac_tpc(ar, ar->hw->conf.chandef.chan);
                if (err)
                        goto out;
        }

out:
        mutex_unlock(&ar->mutex);
        return err;
}

static u64 carl9170_op_prepare_multicast(struct ieee80211_hw *hw,
                                         struct netdev_hw_addr_list *mc_list)
{
        struct netdev_hw_addr *ha;
        u64 mchash;

        /* always get broadcast frames */
        mchash = 1ULL << (0xff >> 2);

        netdev_hw_addr_list_for_each(ha, mc_list)
                mchash |= 1ULL << (ha->addr[5] >> 2);

        return mchash;
}

static void carl9170_op_configure_filter(struct ieee80211_hw *hw,
                                         unsigned int changed_flags,
                                         unsigned int *new_flags,
                                         u64 multicast)
{
        struct ar9170 *ar = hw->priv;

        /* mask supported flags */
        *new_flags &= FIF_ALLMULTI | ar->rx_filter_caps;

        if (!IS_ACCEPTING_CMD(ar))
                return;

        mutex_lock(&ar->mutex);

        ar->filter_state = *new_flags;
        /*
         * We can support more by setting the sniffer bit and
         * then checking the error flags, later.
         */

        if (*new_flags & FIF_ALLMULTI)
                multicast = ~0ULL;

        if (multicast != ar->cur_mc_hash)
                WARN_ON(carl9170_update_multicast(ar, multicast));

        if (changed_flags & FIF_OTHER_BSS) {
                ar->sniffer_enabled = !!(*new_flags & FIF_OTHER_BSS);

                WARN_ON(carl9170_set_operating_mode(ar));
        }

        if (ar->fw.rx_filter && changed_flags & ar->rx_filter_caps) {
                u32 rx_filter = 0;

                if (!ar->fw.ba_filter)
                        rx_filter |= CARL9170_RX_FILTER_CTL_OTHER;

                if (!(*new_flags & (FIF_FCSFAIL | FIF_PLCPFAIL)))
                        rx_filter |= CARL9170_RX_FILTER_BAD;

                if (!(*new_flags & FIF_CONTROL))
                        rx_filter |= CARL9170_RX_FILTER_CTL_OTHER;

                if (!(*new_flags & FIF_PSPOLL))
                        rx_filter |= CARL9170_RX_FILTER_CTL_PSPOLL;

                if (!(*new_flags & FIF_OTHER_BSS)) {
                        rx_filter |= CARL9170_RX_FILTER_OTHER_RA;
                        rx_filter |= CARL9170_RX_FILTER_DECRY_FAIL;
                }

                WARN_ON(carl9170_rx_filter(ar, rx_filter));
        }

        mutex_unlock(&ar->mutex);
}


static void carl9170_op_bss_info_changed(struct ieee80211_hw *hw,
                                         struct ieee80211_vif *vif,
                                         struct ieee80211_bss_conf *bss_conf,
                                         u64 changed)
{
        struct ar9170 *ar = hw->priv;
        struct ath_common *common = &ar->common;
        int err = 0;
        struct carl9170_vif_info *vif_priv;
        struct ieee80211_vif *main_vif;

        mutex_lock(&ar->mutex);
        vif_priv = (void *) vif->drv_priv;
        main_vif = carl9170_get_main_vif(ar);
        if (WARN_ON(!main_vif))
                goto out;

        if (changed & BSS_CHANGED_BEACON_ENABLED) {
                struct carl9170_vif_info *iter;
                int i = 0;

                vif_priv->enable_beacon = bss_conf->enable_beacon;
                rcu_read_lock();
                list_for_each_entry_rcu(iter, &ar->vif_list, list) {
                        if (iter->active && iter->enable_beacon)
                                i++;

                }
                rcu_read_unlock();

                ar->beacon_enabled = i;
        }

        if (changed & BSS_CHANGED_BEACON) {
                err = carl9170_update_beacon(ar, false);
                if (err)
                        goto out;
        }

        if (changed & (BSS_CHANGED_BEACON_ENABLED | BSS_CHANGED_BEACON |
                       BSS_CHANGED_BEACON_INT)) {

                if (main_vif != vif) {
                        bss_conf->beacon_int = main_vif->bss_conf.beacon_int;
                        bss_conf->dtim_period = main_vif->bss_conf.dtim_period;
                }

                /*
                 * Therefore a hard limit for the broadcast traffic should
                 * prevent false alarms.
                 */
                if (vif->type != NL80211_IFTYPE_STATION &&
                    (bss_conf->beacon_int * bss_conf->dtim_period >=
                     (CARL9170_QUEUE_STUCK_TIMEOUT / 2))) {
                        err = -EINVAL;
                        goto out;
                }

                err = carl9170_set_beacon_timers(ar);
                if (err)
                        goto out;
        }

        if (changed & BSS_CHANGED_HT) {
                /* TODO */
                err = 0;
                if (err)
                        goto out;
        }

        if (main_vif != vif)
                goto out;

        /*
         * The following settings can only be changed by the
         * master interface.
         */

        if (changed & BSS_CHANGED_BSSID) {
                memcpy(common->curbssid, bss_conf->bssid, ETH_ALEN);
                err = carl9170_set_operating_mode(ar);
                if (err)
                        goto out;
        }

        if (changed & BSS_CHANGED_ASSOC) {
                ar->common.curaid = vif->cfg.aid;
                err = carl9170_set_beacon_timers(ar);
                if (err)
                        goto out;
        }

        if (changed & BSS_CHANGED_ERP_SLOT) {
                err = carl9170_set_slot_time(ar);
                if (err)
                        goto out;
        }

        if (changed & BSS_CHANGED_BASIC_RATES) {
                err = carl9170_set_mac_rates(ar);
                if (err)
                        goto out;
        }

out:
        WARN_ON_ONCE(err && IS_STARTED(ar));
        mutex_unlock(&ar->mutex);
}

static u64 carl9170_op_get_tsf(struct ieee80211_hw *hw,
                               struct ieee80211_vif *vif)
{
        struct ar9170 *ar = hw->priv;
        struct carl9170_tsf_rsp tsf;
        int err;

        mutex_lock(&ar->mutex);
        err = carl9170_exec_cmd(ar, CARL9170_CMD_READ_TSF,
                                0, NULL, sizeof(tsf), &tsf);
        mutex_unlock(&ar->mutex);
        if (WARN_ON(err))
                return 0;

        return le64_to_cpu(tsf.tsf_64);
}

static int carl9170_op_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
                               struct ieee80211_vif *vif,
                               struct ieee80211_sta *sta,
                               struct ieee80211_key_conf *key)
{
        struct ar9170 *ar = hw->priv;
        int err = 0, i;
        u8 ktype;

        if (ar->disable_offload || !vif)
                return -EOPNOTSUPP;

        /* Fall back to software encryption whenever the driver is connected
         * to more than one network.
         *
         * This is very unfortunate, because some machines cannot handle
         * the high througput speed in 802.11n networks.
         */

        if (!is_main_vif(ar, vif)) {
                mutex_lock(&ar->mutex);
                goto err_softw;
        }

        /*
         * While the hardware supports *catch-all* key, for offloading
         * group-key en-/de-cryption. The way of how the hardware
         * decides which keyId maps to which key, remains a mystery...
         */
        if ((vif->type != NL80211_IFTYPE_STATION &&
             vif->type != NL80211_IFTYPE_ADHOC) &&
            !(key->flags & IEEE80211_KEY_FLAG_PAIRWISE))
                return -EOPNOTSUPP;

        switch (key->cipher) {
        case WLAN_CIPHER_SUITE_WEP40:
                ktype = AR9170_ENC_ALG_WEP64;
                break;
        case WLAN_CIPHER_SUITE_WEP104:
                ktype = AR9170_ENC_ALG_WEP128;
                break;
        case WLAN_CIPHER_SUITE_TKIP:
                ktype = AR9170_ENC_ALG_TKIP;
                break;
        case WLAN_CIPHER_SUITE_CCMP:
                ktype = AR9170_ENC_ALG_AESCCMP;
                key->flags |= IEEE80211_KEY_FLAG_SW_MGMT_TX;
                break;
        default:
                return -EOPNOTSUPP;
        }

        mutex_lock(&ar->mutex);
        if (cmd == SET_KEY) {
                if (!IS_STARTED(ar)) {
                        err = -EOPNOTSUPP;
                        goto out;
                }

                if (!(key->flags & IEEE80211_KEY_FLAG_PAIRWISE)) {
                        sta = NULL;

                        i = 64 + key->keyidx;
                } else {
                        for (i = 0; i < 64; i++)
                                if (!(ar->usedkeys & BIT(i)))
                                        break;
                        if (i == 64)
                                goto err_softw;
                }

                key->hw_key_idx = i;

                err = carl9170_upload_key(ar, i, sta ? sta->addr : NULL,
                                          ktype, 0, key->key,
                                          min_t(u8, 16, key->keylen));
                if (err)
                        goto out;

                if (key->cipher == WLAN_CIPHER_SUITE_TKIP) {
                        err = carl9170_upload_key(ar, i, sta ? sta->addr :
                                                  NULL, ktype, 1,
                                                  key->key + 16, 16);
                        if (err)
                                goto out;

                        /*
                         * hardware is not capable generating MMIC
                         * of fragmented frames!
                         */
                        key->flags |= IEEE80211_KEY_FLAG_GENERATE_MMIC;
                }

                if (i < 64)
                        ar->usedkeys |= BIT(i);

                key->flags |= IEEE80211_KEY_FLAG_GENERATE_IV;
        } else {
                if (!IS_STARTED(ar)) {
                        /* The device is gone... together with the key ;-) */
                        err = 0;
                        goto out;
                }

                if (key->hw_key_idx < 64) {
                        ar->usedkeys &= ~BIT(key->hw_key_idx);
                } else {
                        err = carl9170_upload_key(ar, key->hw_key_idx, NULL,
                                                  AR9170_ENC_ALG_NONE, 0,
                                                  NULL, 0);
                        if (err)
                                goto out;

                        if (key->cipher == WLAN_CIPHER_SUITE_TKIP) {
                                err = carl9170_upload_key(ar, key->hw_key_idx,
                                                          NULL,
                                                          AR9170_ENC_ALG_NONE,
                                                          1, NULL, 0);
                                if (err)
                                        goto out;
                        }

                }

                err = carl9170_disable_key(ar, key->hw_key_idx);
                if (err)
                        goto out;
        }

out:
        mutex_unlock(&ar->mutex);
        return err;

err_softw:
        if (!ar->rx_software_decryption) {
                ar->rx_software_decryption = true;
                carl9170_set_operating_mode(ar);
        }
        mutex_unlock(&ar->mutex);
        return -ENOSPC;
}

static int carl9170_op_sta_add(struct ieee80211_hw *hw,
                               struct ieee80211_vif *vif,
                               struct ieee80211_sta *sta)
{
        struct carl9170_sta_info *sta_info = (void *) sta->drv_priv;
        unsigned int i;

        atomic_set(&sta_info->pending_frames, 0);

        if (sta->deflink.ht_cap.ht_supported) {
                if (sta->deflink.ht_cap.ampdu_density > 6) {
                        /*
                         * HW does support 16us AMPDU density.
                         * No HT-Xmit for station.
                         */

                        return 0;
                }

                for (i = 0; i < ARRAY_SIZE(sta_info->agg); i++)
                        RCU_INIT_POINTER(sta_info->agg[i], NULL);

                sta_info->ampdu_max_len = 1 << (3 + sta->deflink.ht_cap.ampdu_factor);
                sta_info->ht_sta = true;
        }

        return 0;
}

static int carl9170_op_sta_remove(struct ieee80211_hw *hw,
                                struct ieee80211_vif *vif,
                                struct ieee80211_sta *sta)
{
        struct ar9170 *ar = hw->priv;
        struct carl9170_sta_info *sta_info = (void *) sta->drv_priv;
        unsigned int i;
        bool cleanup = false;

        if (sta->deflink.ht_cap.ht_supported) {

                sta_info->ht_sta = false;

                rcu_read_lock();
                for (i = 0; i < ARRAY_SIZE(sta_info->agg); i++) {
                        struct carl9170_sta_tid *tid_info;

                        tid_info = rcu_dereference(sta_info->agg[i]);
                        RCU_INIT_POINTER(sta_info->agg[i], NULL);

                        if (!tid_info)
                                continue;

                        spin_lock_bh(&ar->tx_ampdu_list_lock);
                        if (tid_info->state > CARL9170_TID_STATE_SHUTDOWN)
                                tid_info->state = CARL9170_TID_STATE_SHUTDOWN;
                        spin_unlock_bh(&ar->tx_ampdu_list_lock);
                        cleanup = true;
                }
                rcu_read_unlock();

                if (cleanup)
                        carl9170_ampdu_gc(ar);
        }

        return 0;
}

static int carl9170_op_conf_tx(struct ieee80211_hw *hw,
                               struct ieee80211_vif *vif,
                               unsigned int link_id, u16 queue,
                               const struct ieee80211_tx_queue_params *param)
{
        struct ar9170 *ar = hw->priv;
        int ret;

        mutex_lock(&ar->mutex);
        memcpy(&ar->edcf[ar9170_qmap(queue)], param, sizeof(*param));
        ret = carl9170_set_qos(ar);
        mutex_unlock(&ar->mutex);
        return ret;
}

static void carl9170_ampdu_work(struct work_struct *work)
{
        struct ar9170 *ar = container_of(work, struct ar9170,
                                         ampdu_work);

        if (!IS_STARTED(ar))
                return;

        mutex_lock(&ar->mutex);
        carl9170_ampdu_gc(ar);
        mutex_unlock(&ar->mutex);
}

static int carl9170_op_ampdu_action(struct ieee80211_hw *hw,
                                    struct ieee80211_vif *vif,
                                    struct ieee80211_ampdu_params *params)
{
        struct ieee80211_sta *sta = params->sta;
        enum ieee80211_ampdu_mlme_action action = params->action;
        u16 tid = params->tid;
        u16 *ssn = &params->ssn;
        struct ar9170 *ar = hw->priv;
        struct carl9170_sta_info *sta_info = (void *) sta->drv_priv;
        struct carl9170_sta_tid *tid_info;

        if (modparam_noht)
                return -EOPNOTSUPP;

        switch (action) {
        case IEEE80211_AMPDU_TX_START:
                if (!sta_info->ht_sta)
                        return -EOPNOTSUPP;

                tid_info = kzalloc_obj(struct carl9170_sta_tid);
                if (!tid_info)
                        return -ENOMEM;

                tid_info->hsn = tid_info->bsn = tid_info->snx = (*ssn);
                tid_info->state = CARL9170_TID_STATE_PROGRESS;
                tid_info->tid = tid;
                tid_info->max = sta_info->ampdu_max_len;
                tid_info->sta = sta;
                tid_info->vif = vif;

                INIT_LIST_HEAD(&tid_info->list);
                INIT_LIST_HEAD(&tid_info->tmp_list);
                skb_queue_head_init(&tid_info->queue);
                spin_lock_init(&tid_info->lock);

                spin_lock_bh(&ar->tx_ampdu_list_lock);
                ar->tx_ampdu_list_len++;
                list_add_tail_rcu(&tid_info->list, &ar->tx_ampdu_list);
                rcu_assign_pointer(sta_info->agg[tid], tid_info);
                spin_unlock_bh(&ar->tx_ampdu_list_lock);

                return IEEE80211_AMPDU_TX_START_IMMEDIATE;

        case IEEE80211_AMPDU_TX_STOP_CONT:
        case IEEE80211_AMPDU_TX_STOP_FLUSH:
        case IEEE80211_AMPDU_TX_STOP_FLUSH_CONT:
                rcu_read_lock();
                tid_info = rcu_dereference(sta_info->agg[tid]);
                if (tid_info) {
                        spin_lock_bh(&ar->tx_ampdu_list_lock);
                        if (tid_info->state > CARL9170_TID_STATE_SHUTDOWN)
                                tid_info->state = CARL9170_TID_STATE_SHUTDOWN;
                        spin_unlock_bh(&ar->tx_ampdu_list_lock);
                }

                RCU_INIT_POINTER(sta_info->agg[tid], NULL);
                rcu_read_unlock();

                ieee80211_stop_tx_ba_cb_irqsafe(vif, sta->addr, tid);
                ieee80211_queue_work(ar->hw, &ar->ampdu_work);
                break;

        case IEEE80211_AMPDU_TX_OPERATIONAL:
                rcu_read_lock();
                tid_info = rcu_dereference(sta_info->agg[tid]);

                sta_info->stats[tid].clear = true;
                sta_info->stats[tid].req = false;

                if (tid_info) {
                        bitmap_zero(tid_info->bitmap, CARL9170_BAW_SIZE);
                        tid_info->state = CARL9170_TID_STATE_IDLE;
                }
                rcu_read_unlock();

                if (WARN_ON_ONCE(!tid_info))
                        return -EFAULT;

                break;

        case IEEE80211_AMPDU_RX_START:
        case IEEE80211_AMPDU_RX_STOP:
                /* Handled by hardware */
                break;

        default:
                return -EOPNOTSUPP;
        }

        return 0;
}

#ifdef CONFIG_CARL9170_WPC
static int carl9170_register_wps_button(struct ar9170 *ar)
{
        struct input_dev *input;
        int err;

        if (!(ar->features & CARL9170_WPS_BUTTON))
                return 0;

        input = devm_input_allocate_device(&ar->udev->dev);
        if (!input)
                return -ENOMEM;

        snprintf(ar->wps.name, sizeof(ar->wps.name), "%s WPS Button",
                 wiphy_name(ar->hw->wiphy));

        snprintf(ar->wps.phys, sizeof(ar->wps.phys),
                 "ieee80211/%s/input0", wiphy_name(ar->hw->wiphy));

        input->name = ar->wps.name;
        input->phys = ar->wps.phys;
        input->id.bustype = BUS_USB;
        input->dev.parent = &ar->hw->wiphy->dev;

        input_set_capability(input, EV_KEY, KEY_WPS_BUTTON);

        err = input_register_device(input);
        if (err)
                return err;

        ar->wps.pbc = input;
        return 0;
}
#endif /* CONFIG_CARL9170_WPC */

#ifdef CONFIG_CARL9170_HWRNG
static int carl9170_rng_get(struct ar9170 *ar)
{

#define RW        (CARL9170_MAX_CMD_PAYLOAD_LEN / sizeof(u32))
#define RB        (CARL9170_MAX_CMD_PAYLOAD_LEN)

        static const __le32 rng_load[RW] = {
                [0 ... (RW - 1)] = cpu_to_le32(AR9170_RAND_REG_NUM)};

        u32 buf[RW];

        unsigned int i, off = 0, transfer, count;
        int err;

        BUILD_BUG_ON(RB > CARL9170_MAX_CMD_PAYLOAD_LEN);

        if (!IS_ACCEPTING_CMD(ar))
                return -EAGAIN;

        count = ARRAY_SIZE(ar->rng.cache);
        while (count) {
                err = carl9170_exec_cmd(ar, CARL9170_CMD_RREG,
                                        RB, (u8 *) rng_load,
                                        RB, (u8 *) buf);
                if (err)
                        return err;

                transfer = min_t(unsigned int, count, RW);
                for (i = 0; i < transfer; i++)
                        ar->rng.cache[off + i] = buf[i];

                off += transfer;
                count -= transfer;
        }

        ar->rng.cache_idx = 0;

#undef RW
#undef RB
        return 0;
}

static int carl9170_rng_read(struct hwrng *rng, u32 *data)
{
        struct ar9170 *ar = (struct ar9170 *)rng->priv;
        int ret = -EIO;

        mutex_lock(&ar->mutex);
        if (ar->rng.cache_idx >= ARRAY_SIZE(ar->rng.cache)) {
                ret = carl9170_rng_get(ar);
                if (ret) {
                        mutex_unlock(&ar->mutex);
                        return ret;
                }
        }

        *data = ar->rng.cache[ar->rng.cache_idx++];
        mutex_unlock(&ar->mutex);

        return sizeof(u16);
}

static int carl9170_register_hwrng(struct ar9170 *ar)
{
        int err;

        snprintf(ar->rng.name, ARRAY_SIZE(ar->rng.name),
                 "%s_%s", KBUILD_MODNAME, wiphy_name(ar->hw->wiphy));
        ar->rng.rng.name = ar->rng.name;
        ar->rng.rng.data_read = carl9170_rng_read;
        ar->rng.rng.priv = (unsigned long)ar;

        err = devm_hwrng_register(&ar->udev->dev, &ar->rng.rng);
        if (err) {
                dev_err(&ar->udev->dev, "Failed to register the random "
                        "number generator (%d)\n", err);
                return err;
        }

        return carl9170_rng_get(ar);
}
#endif /* CONFIG_CARL9170_HWRNG */

static int carl9170_op_get_survey(struct ieee80211_hw *hw, int idx,
                                struct survey_info *survey)
{
        struct ar9170 *ar = hw->priv;
        struct ieee80211_channel *chan;
        struct ieee80211_supported_band *band;
        int err, b, i;

        chan = ar->channel;
        if (!chan)
                return -ENODEV;

        if (idx == chan->hw_value) {
                mutex_lock(&ar->mutex);
                err = carl9170_update_survey(ar, false, true);
                mutex_unlock(&ar->mutex);
                if (err)
                        return err;
        }

        for (b = 0; b < NUM_NL80211_BANDS; b++) {
                band = ar->hw->wiphy->bands[b];

                if (!band)
                        continue;

                for (i = 0; i < band->n_channels; i++) {
                        if (band->channels[i].hw_value == idx) {
                                chan = &band->channels[i];
                                goto found;
                        }
                }
        }
        return -ENOENT;

found:
        memcpy(survey, &ar->survey[idx], sizeof(*survey));

        survey->channel = chan;
        survey->filled = SURVEY_INFO_NOISE_DBM;

        if (ar->channel == chan)
                survey->filled |= SURVEY_INFO_IN_USE;

        if (ar->fw.hw_counters) {
                survey->filled |= SURVEY_INFO_TIME |
                                  SURVEY_INFO_TIME_BUSY |
                                  SURVEY_INFO_TIME_TX;
        }

        return 0;
}

static void carl9170_op_flush(struct ieee80211_hw *hw,
                              struct ieee80211_vif *vif,
                              u32 queues, bool drop)
{
        struct ar9170 *ar = hw->priv;
        unsigned int vid;

        mutex_lock(&ar->mutex);
        for_each_set_bit(vid, &ar->vif_bitmap, ar->fw.vif_num)
                carl9170_flush_cab(ar, vid);

        carl9170_flush(ar, drop);
        mutex_unlock(&ar->mutex);
}

static int carl9170_op_get_stats(struct ieee80211_hw *hw,
                                 struct ieee80211_low_level_stats *stats)
{
        struct ar9170 *ar = hw->priv;

        memset(stats, 0, sizeof(*stats));
        stats->dot11ACKFailureCount = ar->tx_ack_failures;
        stats->dot11FCSErrorCount = ar->tx_fcs_errors;
        return 0;
}

static void carl9170_op_sta_notify(struct ieee80211_hw *hw,
                                   struct ieee80211_vif *vif,
                                   enum sta_notify_cmd cmd,
                                   struct ieee80211_sta *sta)
{
        struct carl9170_sta_info *sta_info = (void *) sta->drv_priv;

        switch (cmd) {
        case STA_NOTIFY_SLEEP:
                sta_info->sleeping = true;
                if (atomic_read(&sta_info->pending_frames))
                        ieee80211_sta_block_awake(hw, sta, true);
                break;

        case STA_NOTIFY_AWAKE:
                sta_info->sleeping = false;
                break;
        }
}

static bool carl9170_tx_frames_pending(struct ieee80211_hw *hw)
{
        struct ar9170 *ar = hw->priv;

        return !!atomic_read(&ar->tx_total_queued);
}

static const struct ieee80211_ops carl9170_ops = {
        .add_chanctx = ieee80211_emulate_add_chanctx,
        .remove_chanctx = ieee80211_emulate_remove_chanctx,
        .change_chanctx = ieee80211_emulate_change_chanctx,
        .switch_vif_chanctx = ieee80211_emulate_switch_vif_chanctx,
        .start                        = carl9170_op_start,
        .stop                        = carl9170_op_stop,
        .tx                        = carl9170_op_tx,
        .wake_tx_queue                = ieee80211_handle_wake_tx_queue,
        .flush                        = carl9170_op_flush,
        .add_interface                = carl9170_op_add_interface,
        .remove_interface        = carl9170_op_remove_interface,
        .config                        = carl9170_op_config,
        .prepare_multicast        = carl9170_op_prepare_multicast,
        .configure_filter        = carl9170_op_configure_filter,
        .conf_tx                = carl9170_op_conf_tx,
        .bss_info_changed        = carl9170_op_bss_info_changed,
        .get_tsf                = carl9170_op_get_tsf,
        .set_key                = carl9170_op_set_key,
        .sta_add                = carl9170_op_sta_add,
        .sta_remove                = carl9170_op_sta_remove,
        .sta_notify                = carl9170_op_sta_notify,
        .get_survey                = carl9170_op_get_survey,
        .get_stats                = carl9170_op_get_stats,
        .ampdu_action                = carl9170_op_ampdu_action,
        .tx_frames_pending        = carl9170_tx_frames_pending,
};

void *carl9170_alloc(size_t priv_size)
{
        struct ieee80211_hw *hw;
        struct ar9170 *ar;
        struct sk_buff *skb;
        int i;

        /*
         * this buffer is used for rx stream reconstruction.
         * Under heavy load this device (or the transport layer?)
         * tends to split the streams into separate rx descriptors.
         */

        skb = __dev_alloc_skb(AR9170_RX_STREAM_MAX_SIZE, GFP_KERNEL);
        if (!skb)
                goto err_nomem;

        hw = ieee80211_alloc_hw(priv_size, &carl9170_ops);
        if (!hw)
                goto err_nomem;

        ar = hw->priv;
        ar->hw = hw;
        ar->rx_failover = skb;

        memset(&ar->rx_plcp, 0, sizeof(struct ar9170_rx_head));
        ar->rx_has_plcp = false;

        /*
         * Here's a hidden pitfall!
         *
         * All 4 AC queues work perfectly well under _legacy_ operation.
         * However as soon as aggregation is enabled, the traffic flow
         * gets very bumpy. Therefore we have to _switch_ to a
         * software AC with a single HW queue.
         */
        hw->queues = __AR9170_NUM_TXQ;

        mutex_init(&ar->mutex);
        spin_lock_init(&ar->beacon_lock);
        spin_lock_init(&ar->cmd_lock);
        spin_lock_init(&ar->tx_stats_lock);
        spin_lock_init(&ar->tx_ampdu_list_lock);
        spin_lock_init(&ar->mem_lock);
        spin_lock_init(&ar->state_lock);
        atomic_set(&ar->pending_restarts, 0);
        ar->vifs = 0;
        for (i = 0; i < ar->hw->queues; i++) {
                skb_queue_head_init(&ar->tx_status[i]);
                skb_queue_head_init(&ar->tx_pending[i]);

                INIT_LIST_HEAD(&ar->bar_list[i]);
                spin_lock_init(&ar->bar_list_lock[i]);
        }
        INIT_WORK(&ar->ps_work, carl9170_ps_work);
        INIT_WORK(&ar->ping_work, carl9170_ping_work);
        INIT_WORK(&ar->restart_work, carl9170_restart_work);
        INIT_WORK(&ar->ampdu_work, carl9170_ampdu_work);
        INIT_DELAYED_WORK(&ar->stat_work, carl9170_stat_work);
        INIT_DELAYED_WORK(&ar->tx_janitor, carl9170_tx_janitor);
        INIT_LIST_HEAD(&ar->tx_ampdu_list);
        rcu_assign_pointer(ar->tx_ampdu_iter,
                           (struct carl9170_sta_tid *) &ar->tx_ampdu_list);

        bitmap_zero(&ar->vif_bitmap, ar->fw.vif_num);
        INIT_LIST_HEAD(&ar->vif_list);
        init_completion(&ar->tx_flush);

        /* firmware decides which modes we support */
        hw->wiphy->interface_modes = 0;

        ieee80211_hw_set(hw, RX_INCLUDES_FCS);
        ieee80211_hw_set(hw, MFP_CAPABLE);
        ieee80211_hw_set(hw, REPORTS_TX_ACK_STATUS);
        ieee80211_hw_set(hw, SUPPORTS_PS);
        ieee80211_hw_set(hw, PS_NULLFUNC_STACK);
        ieee80211_hw_set(hw, NEED_DTIM_BEFORE_ASSOC);
        ieee80211_hw_set(hw, SUPPORTS_RC_TABLE);
        ieee80211_hw_set(hw, SIGNAL_DBM);
        ieee80211_hw_set(hw, SUPPORTS_HT_CCK_RATES);

        if (!modparam_noht) {
                /*
                 * see the comment above, why we allow the user
                 * to disable HT by a module parameter.
                 */
                ieee80211_hw_set(hw, AMPDU_AGGREGATION);
        }

        hw->extra_tx_headroom = sizeof(struct _carl9170_tx_superframe);
        hw->sta_data_size = sizeof(struct carl9170_sta_info);
        hw->vif_data_size = sizeof(struct carl9170_vif_info);

        hw->max_rates = CARL9170_TX_MAX_RATES;
        hw->max_rate_tries = CARL9170_TX_USER_RATE_TRIES;

        for (i = 0; i < ARRAY_SIZE(ar->noise); i++)
                ar->noise[i] = -95; /* ATH_DEFAULT_NOISE_FLOOR */

        wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_CQM_RSSI_LIST);

        return ar;

err_nomem:
        kfree_skb(skb);
        return ERR_PTR(-ENOMEM);
}

static int carl9170_read_eeprom(struct ar9170 *ar)
{
#define RW        8        /* number of words to read at once */
#define RB        (sizeof(u32) * RW)
        u8 *eeprom = (void *)&ar->eeprom;
        __le32 offsets[RW];
        int i, j, err;

        BUILD_BUG_ON(sizeof(ar->eeprom) & 3);

        BUILD_BUG_ON(RB > CARL9170_MAX_CMD_LEN - 4);
#ifndef __CHECKER__
        /* don't want to handle trailing remains */
        BUILD_BUG_ON(sizeof(ar->eeprom) % RB);
#endif

        for (i = 0; i < sizeof(ar->eeprom) / RB; i++) {
                for (j = 0; j < RW; j++)
                        offsets[j] = cpu_to_le32(AR9170_EEPROM_START +
                                                 RB * i + 4 * j);

                err = carl9170_exec_cmd(ar, CARL9170_CMD_RREG,
                                        RB, (u8 *) &offsets,
                                        RB, eeprom + RB * i);
                if (err)
                        return err;
        }

#undef RW
#undef RB
        return 0;
}

static int carl9170_parse_eeprom(struct ar9170 *ar)
{
        struct ath_regulatory *regulatory = &ar->common.regulatory;
        unsigned int rx_streams, tx_streams, tx_params = 0;
        int bands = 0;
        int chans = 0;

        if (ar->eeprom.length == cpu_to_le16(0xffff))
                return -ENODATA;

        rx_streams = hweight8(ar->eeprom.rx_mask);
        tx_streams = hweight8(ar->eeprom.tx_mask);

        if (rx_streams != tx_streams) {
                tx_params = IEEE80211_HT_MCS_TX_RX_DIFF;

                WARN_ON(!(tx_streams >= 1 && tx_streams <=
                        IEEE80211_HT_MCS_TX_MAX_STREAMS));

                tx_params |= (tx_streams - 1) <<
                            IEEE80211_HT_MCS_TX_MAX_STREAMS_SHIFT;

                carl9170_band_2GHz.ht_cap.mcs.tx_params |= tx_params;
                carl9170_band_5GHz.ht_cap.mcs.tx_params |= tx_params;
        }

        if (ar->eeprom.operating_flags & AR9170_OPFLAG_2GHZ) {
                ar->hw->wiphy->bands[NL80211_BAND_2GHZ] =
                        &carl9170_band_2GHz;
                chans += carl9170_band_2GHz.n_channels;
                bands++;
        }
        if (ar->eeprom.operating_flags & AR9170_OPFLAG_5GHZ) {
                ar->hw->wiphy->bands[NL80211_BAND_5GHZ] =
                        &carl9170_band_5GHz;
                chans += carl9170_band_5GHz.n_channels;
                bands++;
        }

        if (!bands)
                return -EINVAL;

        ar->survey = devm_kcalloc(&ar->udev->dev, chans,
                                  sizeof(struct survey_info), GFP_KERNEL);
        if (!ar->survey)
                return -ENOMEM;
        ar->num_channels = chans;

        regulatory->current_rd = le16_to_cpu(ar->eeprom.reg_domain[0]);

        /* second part of wiphy init */
        SET_IEEE80211_PERM_ADDR(ar->hw, ar->eeprom.mac_address);

        return 0;
}

static void carl9170_reg_notifier(struct wiphy *wiphy,
                                  struct regulatory_request *request)
{
        struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy);
        struct ar9170 *ar = hw->priv;

        ath_reg_notifier_apply(wiphy, request, &ar->common.regulatory);
}

int carl9170_register(struct ar9170 *ar)
{
        struct ath_regulatory *regulatory = &ar->common.regulatory;
        int err = 0, i;

        ar->mem_bitmap = devm_bitmap_zalloc(&ar->udev->dev, ar->fw.mem_blocks, GFP_KERNEL);
        if (!ar->mem_bitmap)
                return -ENOMEM;

        /* try to read EEPROM, init MAC addr */
        err = carl9170_read_eeprom(ar);
        if (err)
                return err;

        err = carl9170_parse_eeprom(ar);
        if (err)
                return err;

        err = ath_regd_init(regulatory, ar->hw->wiphy,
                            carl9170_reg_notifier);
        if (err)
                return err;

        if (modparam_noht) {
                carl9170_band_2GHz.ht_cap.ht_supported = false;
                carl9170_band_5GHz.ht_cap.ht_supported = false;
        }

        for (i = 0; i < ar->fw.vif_num; i++) {
                ar->vif_priv[i].id = i;
                ar->vif_priv[i].vif = NULL;
        }

        err = ieee80211_register_hw(ar->hw);
        if (err)
                return err;

        /* mac80211 interface is now registered */
        ar->registered = true;

        if (!ath_is_world_regd(regulatory))
                regulatory_hint(ar->hw->wiphy, regulatory->alpha2);

#ifdef CONFIG_CARL9170_DEBUGFS
        carl9170_debugfs_register(ar);
#endif /* CONFIG_CARL9170_DEBUGFS */

        err = carl9170_led_init(ar);
        if (err)
                goto err_unreg;

#ifdef CONFIG_CARL9170_LEDS
        err = carl9170_led_register(ar);
        if (err)
                goto err_unreg;
#endif /* CONFIG_CARL9170_LEDS */

#ifdef CONFIG_CARL9170_WPC
        err = carl9170_register_wps_button(ar);
        if (err)
                goto err_unreg;
#endif /* CONFIG_CARL9170_WPC */

#ifdef CONFIG_CARL9170_HWRNG
        err = carl9170_register_hwrng(ar);
        if (err)
                goto err_unreg;
#endif /* CONFIG_CARL9170_HWRNG */

        dev_info(&ar->udev->dev, "Atheros AR9170 is registered as '%s'\n",
                 wiphy_name(ar->hw->wiphy));

        return 0;

err_unreg:
        carl9170_unregister(ar);
        return err;
}

void carl9170_unregister(struct ar9170 *ar)
{
        if (!ar->registered)
                return;

        ar->registered = false;

#ifdef CONFIG_CARL9170_LEDS
        carl9170_led_unregister(ar);
#endif /* CONFIG_CARL9170_LEDS */

#ifdef CONFIG_CARL9170_DEBUGFS
        carl9170_debugfs_unregister(ar);
#endif /* CONFIG_CARL9170_DEBUGFS */

        carl9170_cancel_worker(ar);
        cancel_work_sync(&ar->restart_work);

        ieee80211_unregister_hw(ar->hw);
}

void carl9170_free(struct ar9170 *ar)
{
        WARN_ON(ar->registered);
        WARN_ON(IS_INITIALIZED(ar));

        kfree_skb(ar->rx_failover);
        ar->rx_failover = NULL;

        mutex_destroy(&ar->mutex);

        ieee80211_free_hw(ar->hw);
}






























































































































































































    1 




























    4 





















  197 
























  374 





















  386 









































    3 





















    1 






















    1 
































    1 






    1 











































    1 







    1 









































    1 





























    1 





























   56 


























  163 


































    2 






























    2 






    1 



























  100 



































    1 




























    1 















    1 

















































    1 











































    3 







































  324 









































    1 






























   80 


























   81 






















   35 




















   24 






















  315 


































































   79 







































    1 































    1 










    1 





































    1 


































    1 





























































    1 




















  259 






  259 






  287 






  146 







































































   19 




























   38 
























   38 






















    1 





















  230 






  230 







    9 





































    3 









































    3 



























    1 






    2 






    3 



































  448 






  197 































  516 



















    4 





























    3 













































































































































   68 




























  412 





























   75 














































  106 


































































  108 





























  106 











































    3 
































  283 





    2 





    3 


























  224 





















    1 





























  709 





















    4 







































































































































































   33 












































































































   12 


    1 



















  349 











































































































































































































































































































  614 




























  287 



























































   22 
























































    7 











































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM ext4

#if !defined(_TRACE_EXT4_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_EXT4_H

#include <linux/writeback.h>
#include <linux/tracepoint.h>

struct ext4_allocation_context;
struct ext4_allocation_request;
struct ext4_extent;
struct ext4_prealloc_space;
struct ext4_inode_info;
struct mpage_da_data;
struct ext4_map_blocks;
struct extent_status;
struct ext4_fsmap;
struct partial_cluster;

#define EXT4_I(inode) (container_of(inode, struct ext4_inode_info, vfs_inode))

#define show_mballoc_flags(flags) __print_flags(flags, "|",        \
        { EXT4_MB_HINT_MERGE,                "HINT_MERGE" },                \
        { EXT4_MB_HINT_FIRST,                "HINT_FIRST" },                \
        { EXT4_MB_HINT_DATA,                "HINT_DATA" },                \
        { EXT4_MB_HINT_NOPREALLOC,        "HINT_NOPREALLOC" },        \
        { EXT4_MB_HINT_GROUP_ALLOC,        "HINT_GRP_ALLOC" },        \
        { EXT4_MB_HINT_GOAL_ONLY,        "HINT_GOAL_ONLY" },        \
        { EXT4_MB_HINT_TRY_GOAL,        "HINT_TRY_GOAL" },        \
        { EXT4_MB_DELALLOC_RESERVED,        "DELALLOC_RESV" },        \
        { EXT4_MB_STREAM_ALLOC,                "STREAM_ALLOC" },        \
        { EXT4_MB_USE_ROOT_BLOCKS,        "USE_ROOT_BLKS" },        \
        { EXT4_MB_USE_RESERVED,                "USE_RESV" },                \
        { EXT4_MB_STRICT_CHECK,                "STRICT_CHECK" })

#define show_map_flags(flags) __print_flags(flags, "|",                        \
        { EXT4_GET_BLOCKS_CREATE,                "CREATE" },                \
        { EXT4_GET_BLOCKS_UNWRIT_EXT,                "UNWRIT" },                \
        { EXT4_GET_BLOCKS_DELALLOC_RESERVE,        "DELALLOC" },                \
        { EXT4_GET_BLOCKS_SPLIT_NOMERGE,        "SPLIT_NOMERGE" },        \
        { EXT4_GET_BLOCKS_CONVERT,                "CONVERT" },                \
        { EXT4_GET_BLOCKS_METADATA_NOFAIL,        "METADATA_NOFAIL" },        \
        { EXT4_GET_BLOCKS_NO_NORMALIZE,                "NO_NORMALIZE" },        \
        { EXT4_GET_BLOCKS_CONVERT_UNWRITTEN,        "CONVERT_UNWRITTEN" },  \
        { EXT4_GET_BLOCKS_ZERO,                        "ZERO" },                \
        { EXT4_GET_BLOCKS_IO_SUBMIT,                "IO_SUBMIT" },                \
        { EXT4_EX_NOCACHE,                        "EX_NOCACHE" })

/*
 * __print_flags() requires that all enum values be wrapped in the
 * TRACE_DEFINE_ENUM macro so that the enum value can be encoded in the ftrace
 * ring buffer.
 */
TRACE_DEFINE_ENUM(BH_New);
TRACE_DEFINE_ENUM(BH_Mapped);
TRACE_DEFINE_ENUM(BH_Unwritten);
TRACE_DEFINE_ENUM(BH_Boundary);

#define show_mflags(flags) __print_flags(flags, "",        \
        { EXT4_MAP_NEW,                "N" },                        \
        { EXT4_MAP_MAPPED,        "M" },                        \
        { EXT4_MAP_UNWRITTEN,        "U" },                        \
        { EXT4_MAP_BOUNDARY,        "B" })

#define show_free_flags(flags) __print_flags(flags, "|",        \
        { EXT4_FREE_BLOCKS_METADATA,                "METADATA" },        \
        { EXT4_FREE_BLOCKS_FORGET,                "FORGET" },        \
        { EXT4_FREE_BLOCKS_VALIDATED,                "VALIDATED" },        \
        { EXT4_FREE_BLOCKS_NO_QUOT_UPDATE,        "NO_QUOTA" },        \
        { EXT4_FREE_BLOCKS_NOFREE_FIRST_CLUSTER,"1ST_CLUSTER" },\
        { EXT4_FREE_BLOCKS_NOFREE_LAST_CLUSTER,        "LAST_CLUSTER" })

TRACE_DEFINE_ENUM(ES_WRITTEN_B);
TRACE_DEFINE_ENUM(ES_UNWRITTEN_B);
TRACE_DEFINE_ENUM(ES_DELAYED_B);
TRACE_DEFINE_ENUM(ES_HOLE_B);
TRACE_DEFINE_ENUM(ES_REFERENCED_B);

#define show_extent_status(status) __print_flags(status, "",        \
        { EXTENT_STATUS_WRITTEN,        "W" },                        \
        { EXTENT_STATUS_UNWRITTEN,        "U" },                        \
        { EXTENT_STATUS_DELAYED,        "D" },                        \
        { EXTENT_STATUS_HOLE,                "H" },                        \
        { EXTENT_STATUS_REFERENCED,        "R" })

#define show_falloc_mode(mode) __print_flags(mode, "|",                \
        { FALLOC_FL_KEEP_SIZE,                "KEEP_SIZE"},                \
        { FALLOC_FL_PUNCH_HOLE,                "PUNCH_HOLE"},                \
        { FALLOC_FL_COLLAPSE_RANGE,        "COLLAPSE_RANGE"},        \
        { FALLOC_FL_ZERO_RANGE,                "ZERO_RANGE"},                \
        { FALLOC_FL_WRITE_ZEROES,        "WRITE_ZEROES"})

TRACE_DEFINE_ENUM(EXT4_FC_REASON_XATTR);
TRACE_DEFINE_ENUM(EXT4_FC_REASON_CROSS_RENAME);
TRACE_DEFINE_ENUM(EXT4_FC_REASON_JOURNAL_FLAG_CHANGE);
TRACE_DEFINE_ENUM(EXT4_FC_REASON_NOMEM);
TRACE_DEFINE_ENUM(EXT4_FC_REASON_SWAP_BOOT);
TRACE_DEFINE_ENUM(EXT4_FC_REASON_RESIZE);
TRACE_DEFINE_ENUM(EXT4_FC_REASON_RENAME_DIR);
TRACE_DEFINE_ENUM(EXT4_FC_REASON_FALLOC_RANGE);
TRACE_DEFINE_ENUM(EXT4_FC_REASON_INODE_JOURNAL_DATA);
TRACE_DEFINE_ENUM(EXT4_FC_REASON_ENCRYPTED_FILENAME);
TRACE_DEFINE_ENUM(EXT4_FC_REASON_MIGRATE);
TRACE_DEFINE_ENUM(EXT4_FC_REASON_VERITY);
TRACE_DEFINE_ENUM(EXT4_FC_REASON_MOVE_EXT);
TRACE_DEFINE_ENUM(EXT4_FC_REASON_MAX);

#undef EM
#undef EMe
#define EM(a)        TRACE_DEFINE_ENUM(EXT4_FC_SNAP_ERR_##a);
#define EMe(a)        TRACE_DEFINE_ENUM(EXT4_FC_SNAP_ERR_##a);

#define TRACE_SNAP_ERR                                                \
        EM(NONE)                                                \
        EM(ES_MISS)                                                \
        EM(ES_DELAYED)                                                \
        EM(ES_OTHER)                                                \
        EM(INODES_CAP)                                                \
        EM(RANGES_CAP)                                                \
        EM(NOMEM)                                                \
        EMe(INODE_LOC)

TRACE_SNAP_ERR

#undef EM
#undef EMe

#define show_fc_reason(reason)                                                \
        __print_symbolic(reason,                                        \
                { EXT4_FC_REASON_XATTR,                "XATTR"},                \
                { EXT4_FC_REASON_CROSS_RENAME,        "CROSS_RENAME"},        \
                { EXT4_FC_REASON_JOURNAL_FLAG_CHANGE, "JOURNAL_FLAG_CHANGE"}, \
                { EXT4_FC_REASON_NOMEM,        "NO_MEM"},                        \
                { EXT4_FC_REASON_SWAP_BOOT,        "SWAP_BOOT"},                \
                { EXT4_FC_REASON_RESIZE,        "RESIZE"},                \
                { EXT4_FC_REASON_RENAME_DIR,        "RENAME_DIR"},                \
                { EXT4_FC_REASON_FALLOC_RANGE,        "FALLOC_RANGE"},        \
                { EXT4_FC_REASON_INODE_JOURNAL_DATA,        "INODE_JOURNAL_DATA"}, \
                { EXT4_FC_REASON_ENCRYPTED_FILENAME,        "ENCRYPTED_FILENAME"}, \
                { EXT4_FC_REASON_MIGRATE,                "MIGRATE"},        \
                { EXT4_FC_REASON_VERITY,                "VERITY"},        \
                { EXT4_FC_REASON_MOVE_EXT,                "MOVE_EXT"})

TRACE_DEFINE_ENUM(CR_POWER2_ALIGNED);
TRACE_DEFINE_ENUM(CR_GOAL_LEN_FAST);
TRACE_DEFINE_ENUM(CR_BEST_AVAIL_LEN);
TRACE_DEFINE_ENUM(CR_GOAL_LEN_SLOW);
TRACE_DEFINE_ENUM(CR_ANY_FREE);

#define show_criteria(cr)                                               \
        __print_symbolic(cr,                                            \
                         { CR_POWER2_ALIGNED, "CR_POWER2_ALIGNED" },        \
                         { CR_GOAL_LEN_FAST, "CR_GOAL_LEN_FAST" },      \
                         { CR_BEST_AVAIL_LEN, "CR_BEST_AVAIL_LEN" },    \
                         { CR_GOAL_LEN_SLOW, "CR_GOAL_LEN_SLOW" },      \
                         { CR_ANY_FREE, "CR_ANY_FREE" })

TRACE_EVENT(ext4_other_inode_update_time,
        TP_PROTO(struct inode *inode, u64 orig_ino),

        TP_ARGS(inode, orig_ino),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        u64,        orig_ino                )
                __field(        dev_t,        dev                        )
                __field(        uid_t,        uid                        )
                __field(        gid_t,        gid                        )
                __field(        __u16, mode                        )
        ),

        TP_fast_assign(
                __entry->orig_ino = orig_ino;
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->uid        = i_uid_read(inode);
                __entry->gid        = i_gid_read(inode);
                __entry->mode        = inode->i_mode;
        ),

        TP_printk("dev %d,%d orig_ino %llu ino %llu mode 0%o uid %u gid %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->orig_ino,
                  __entry->ino, __entry->mode,
                  __entry->uid, __entry->gid)
);

TRACE_EVENT(ext4_free_inode,
        TP_PROTO(struct inode *inode),

        TP_ARGS(inode),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        __u64, blocks                        )
                __field(        dev_t,        dev                        )
                __field(        uid_t,        uid                        )
                __field(        gid_t,        gid                        )
                __field(        __u16, mode                        )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->uid        = i_uid_read(inode);
                __entry->gid        = i_gid_read(inode);
                __entry->blocks        = inode->i_blocks;
                __entry->mode        = inode->i_mode;
        ),

        TP_printk("dev %d,%d ino %llu mode 0%o uid %u gid %u blocks %llu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino, __entry->mode,
                  __entry->uid, __entry->gid, __entry->blocks)
);

TRACE_EVENT(ext4_request_inode,
        TP_PROTO(struct inode *dir, int mode),

        TP_ARGS(dir, mode),

        TP_STRUCT__entry(
                __field(        u64,        dir                        )
                __field(        dev_t,        dev                        )
                __field(        __u16, mode                        )
        ),

        TP_fast_assign(
                __entry->dev        = dir->i_sb->s_dev;
                __entry->dir        = dir->i_ino;
                __entry->mode        = mode;
        ),

        TP_printk("dev %d,%d dir %llu mode 0%o",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->dir, __entry->mode)
);

TRACE_EVENT(ext4_allocate_inode,
        TP_PROTO(struct inode *inode, struct inode *dir, int mode),

        TP_ARGS(inode, dir, mode),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        u64,        dir                        )
                __field(        dev_t,        dev                        )
                __field(        __u16,        mode                        )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->dir        = dir->i_ino;
                __entry->mode        = mode;
        ),

        TP_printk("dev %d,%d ino %llu dir %llu mode 0%o",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->dir, __entry->mode)
);

TRACE_EVENT(ext4_evict_inode,
        TP_PROTO(struct inode *inode),

        TP_ARGS(inode),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        dev_t,        dev                        )
                __field(        int,        nlink                        )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->nlink        = inode->i_nlink;
        ),

        TP_printk("dev %d,%d ino %llu nlink %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino, __entry->nlink)
);

TRACE_EVENT(ext4_drop_inode,
        TP_PROTO(struct inode *inode, int drop),

        TP_ARGS(inode, drop),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        dev_t,        dev                        )
                __field(        int,        drop                        )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->drop        = drop;
        ),

        TP_printk("dev %d,%d ino %llu drop %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino, __entry->drop)
);

TRACE_EVENT(ext4_nfs_commit_metadata,
        TP_PROTO(struct inode *inode),

        TP_ARGS(inode),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        dev_t,        dev                        )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
        ),

        TP_printk("dev %d,%d ino %llu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino)
);

TRACE_EVENT(ext4_mark_inode_dirty,
        TP_PROTO(struct inode *inode, unsigned long IP),

        TP_ARGS(inode, IP),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(unsigned long,        ip                        )
                __field(        dev_t,        dev                        )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->ip        = IP;
        ),

        TP_printk("dev %d,%d ino %llu caller %pS",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino, (void *)__entry->ip)
);

TRACE_EVENT(ext4_begin_ordered_truncate,
        TP_PROTO(struct inode *inode, loff_t new_size),

        TP_ARGS(inode, new_size),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        loff_t,        new_size                )
                __field(        dev_t,        dev                        )
        ),

        TP_fast_assign(
                __entry->dev                = inode->i_sb->s_dev;
                __entry->ino                = inode->i_ino;
                __entry->new_size        = new_size;
        ),

        TP_printk("dev %d,%d ino %llu new_size %lld",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->new_size)
);

DECLARE_EVENT_CLASS(ext4__write_begin,

        TP_PROTO(struct inode *inode, loff_t pos, unsigned int len),

        TP_ARGS(inode, pos, len),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        loff_t,        pos                        )
                __field(        dev_t,        dev                        )
                __field(        unsigned int, len                )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->pos        = pos;
                __entry->len        = len;
        ),

        TP_printk("dev %d,%d ino %llu pos %lld len %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->pos, __entry->len)
);

DEFINE_EVENT(ext4__write_begin, ext4_write_begin,

        TP_PROTO(struct inode *inode, loff_t pos, unsigned int len),

        TP_ARGS(inode, pos, len)
);

DEFINE_EVENT(ext4__write_begin, ext4_da_write_begin,

        TP_PROTO(struct inode *inode, loff_t pos, unsigned int len),

        TP_ARGS(inode, pos, len)
);

DECLARE_EVENT_CLASS(ext4__write_end,
        TP_PROTO(struct inode *inode, loff_t pos, unsigned int len,
                        unsigned int copied),

        TP_ARGS(inode, pos, len, copied),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        loff_t,        pos                        )
                __field(        dev_t,        dev                        )
                __field(        unsigned int, len                )
                __field(        unsigned int, copied                )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->pos        = pos;
                __entry->len        = len;
                __entry->copied        = copied;
        ),

        TP_printk("dev %d,%d ino %llu pos %lld len %u copied %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->pos, __entry->len, __entry->copied)
);

DEFINE_EVENT(ext4__write_end, ext4_write_end,

        TP_PROTO(struct inode *inode, loff_t pos, unsigned int len,
                 unsigned int copied),

        TP_ARGS(inode, pos, len, copied)
);

DEFINE_EVENT(ext4__write_end, ext4_journalled_write_end,

        TP_PROTO(struct inode *inode, loff_t pos, unsigned int len,
                 unsigned int copied),

        TP_ARGS(inode, pos, len, copied)
);

DEFINE_EVENT(ext4__write_end, ext4_da_write_end,

        TP_PROTO(struct inode *inode, loff_t pos, unsigned int len,
                 unsigned int copied),

        TP_ARGS(inode, pos, len, copied)
);

TRACE_EVENT(ext4_writepages,
        TP_PROTO(struct inode *inode, struct writeback_control *wbc),

        TP_ARGS(inode, wbc),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        loff_t,        range_start                )
                __field(        loff_t,        range_end                )
                __field(        long,        nr_to_write                )
                __field(        long,        pages_skipped                )
                __field(       pgoff_t,        writeback_index                )
                __field(        dev_t,        dev                        )
                __field(        int,        sync_mode                )
                __field(        char,        for_kupdate                )
                __field(        char,        range_cyclic                )
        ),

        TP_fast_assign(
                __entry->dev                = inode->i_sb->s_dev;
                __entry->ino                = inode->i_ino;
                __entry->nr_to_write        = wbc->nr_to_write;
                __entry->pages_skipped        = wbc->pages_skipped;
                __entry->range_start        = wbc->range_start;
                __entry->range_end        = wbc->range_end;
                __entry->writeback_index = inode->i_mapping->writeback_index;
                __entry->sync_mode        = wbc->sync_mode;
                __entry->for_kupdate        = wbc->for_kupdate;
                __entry->range_cyclic        = wbc->range_cyclic;
        ),

        TP_printk("dev %d,%d ino %llu nr_to_write %ld pages_skipped %ld "
                  "range_start %lld range_end %lld sync_mode %d "
                  "for_kupdate %d range_cyclic %d writeback_index %lu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino, __entry->nr_to_write,
                  __entry->pages_skipped, __entry->range_start,
                  __entry->range_end, __entry->sync_mode,
                  __entry->for_kupdate, __entry->range_cyclic,
                  (unsigned long) __entry->writeback_index)
);

TRACE_EVENT(ext4_da_write_folios_start,
        TP_PROTO(struct inode *inode, loff_t start_pos, loff_t next_pos,
                 struct writeback_control *wbc),

        TP_ARGS(inode, start_pos, next_pos, wbc),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(       loff_t,        start_pos                )
                __field(       loff_t,        next_pos                )
                __field(         long,        nr_to_write                )
                __field(        dev_t,        dev                        )
                __field(          int,        sync_mode                )
        ),

        TP_fast_assign(
                __entry->dev                = inode->i_sb->s_dev;
                __entry->ino                = inode->i_ino;
                __entry->start_pos        = start_pos;
                __entry->next_pos        = next_pos;
                __entry->nr_to_write        = wbc->nr_to_write;
                __entry->sync_mode        = wbc->sync_mode;
        ),

        TP_printk("dev %d,%d ino %llu start_pos 0x%llx next_pos 0x%llx nr_to_write %ld sync_mode %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino, __entry->start_pos, __entry->next_pos,
                  __entry->nr_to_write, __entry->sync_mode)
);

TRACE_EVENT(ext4_da_write_folios_end,
        TP_PROTO(struct inode *inode, loff_t start_pos, loff_t next_pos,
                 struct writeback_control *wbc, int ret),

        TP_ARGS(inode, start_pos, next_pos, wbc, ret),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(       loff_t,        start_pos                )
                __field(       loff_t,        next_pos                )
                __field(         long,        nr_to_write                )
                __field(        dev_t,        dev                        )
                __field(          int,        ret                        )
        ),

        TP_fast_assign(
                __entry->dev                = inode->i_sb->s_dev;
                __entry->ino                = inode->i_ino;
                __entry->start_pos        = start_pos;
                __entry->next_pos        = next_pos;
                __entry->nr_to_write        = wbc->nr_to_write;
                __entry->ret                = ret;
        ),

        TP_printk("dev %d,%d ino %llu start_pos 0x%llx next_pos 0x%llx nr_to_write %ld ret %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino, __entry->start_pos, __entry->next_pos,
                  __entry->nr_to_write, __entry->ret)
);

TRACE_EVENT(ext4_da_write_pages_extent,
        TP_PROTO(struct inode *inode, struct ext4_map_blocks *map),

        TP_ARGS(inode, map),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        __u64,        lblk                        )
                __field(        dev_t,        dev                        )
                __field(        __u32,        len                        )
                __field(        __u32,        flags                        )
        ),

        TP_fast_assign(
                __entry->dev                = inode->i_sb->s_dev;
                __entry->ino                = inode->i_ino;
                __entry->lblk                = map->m_lblk;
                __entry->len                = map->m_len;
                __entry->flags                = map->m_flags;
        ),

        TP_printk("dev %d,%d ino %llu lblk %llu len %u flags %s",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino, __entry->lblk, __entry->len,
                  show_mflags(__entry->flags))
);

TRACE_EVENT(ext4_writepages_result,
        TP_PROTO(struct inode *inode, struct writeback_control *wbc,
                        int ret, int pages_written),

        TP_ARGS(inode, wbc, ret, pages_written),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        long,        pages_skipped                )
                __field(       pgoff_t,        writeback_index                )
                __field(        dev_t,        dev                        )
                __field(        int,        ret                        )
                __field(        int,        pages_written                )
                __field(        int,        sync_mode                )
        ),

        TP_fast_assign(
                __entry->dev                = inode->i_sb->s_dev;
                __entry->ino                = inode->i_ino;
                __entry->ret                = ret;
                __entry->pages_written        = pages_written;
                __entry->pages_skipped        = wbc->pages_skipped;
                __entry->writeback_index = inode->i_mapping->writeback_index;
                __entry->sync_mode        = wbc->sync_mode;
        ),

        TP_printk("dev %d,%d ino %llu ret %d pages_written %d pages_skipped %ld "
                  "sync_mode %d writeback_index %lu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino, __entry->ret,
                  __entry->pages_written, __entry->pages_skipped,
                  __entry->sync_mode,
                  (unsigned long) __entry->writeback_index)
);

DECLARE_EVENT_CLASS(ext4__folio_op,
        TP_PROTO(struct inode *inode, struct folio *folio),

        TP_ARGS(inode, folio),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        pgoff_t, index                        )
                __field(        dev_t,        dev                        )

        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->index        = folio->index;
        ),

        TP_printk("dev %d,%d ino %llu folio_index %lu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  (unsigned long) __entry->index)
);

DEFINE_EVENT(ext4__folio_op, ext4_read_folio,

        TP_PROTO(struct inode *inode, struct folio *folio),

        TP_ARGS(inode, folio)
);

DEFINE_EVENT(ext4__folio_op, ext4_release_folio,

        TP_PROTO(struct inode *inode, struct folio *folio),

        TP_ARGS(inode, folio)
);

DECLARE_EVENT_CLASS(ext4_invalidate_folio_op,
        TP_PROTO(struct folio *folio, size_t offset, size_t length),

        TP_ARGS(folio, offset, length),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        pgoff_t, index                        )
                __field(        size_t, offset                        )
                __field(        size_t, length                        )
                __field(        dev_t,        dev                        )
        ),

        TP_fast_assign(
                __entry->dev        = folio->mapping->host->i_sb->s_dev;
                __entry->ino        = folio->mapping->host->i_ino;
                __entry->index        = folio->index;
                __entry->offset        = offset;
                __entry->length        = length;
        ),

        TP_printk("dev %d,%d ino %llu folio_index %lu offset %zu length %zu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  (unsigned long) __entry->index,
                  __entry->offset, __entry->length)
);

DEFINE_EVENT(ext4_invalidate_folio_op, ext4_invalidate_folio,
        TP_PROTO(struct folio *folio, size_t offset, size_t length),

        TP_ARGS(folio, offset, length)
);

DEFINE_EVENT(ext4_invalidate_folio_op, ext4_journalled_invalidate_folio,
        TP_PROTO(struct folio *folio, size_t offset, size_t length),

        TP_ARGS(folio, offset, length)
);

TRACE_EVENT(ext4_discard_blocks,
        TP_PROTO(struct super_block *sb, unsigned long long blk,
                        unsigned long long count),

        TP_ARGS(sb, blk, count),

        TP_STRUCT__entry(
                __field(        dev_t,        dev                        )
                __field(        __u64,        blk                        )
                __field(        __u64,        count                        )

        ),

        TP_fast_assign(
                __entry->dev        = sb->s_dev;
                __entry->blk        = blk;
                __entry->count        = count;
        ),

        TP_printk("dev %d,%d blk %llu count %llu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->blk, __entry->count)
);

DECLARE_EVENT_CLASS(ext4__mb_new_pa,
        TP_PROTO(struct ext4_allocation_context *ac,
                 struct ext4_prealloc_space *pa),

        TP_ARGS(ac, pa),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        __u64,        pa_pstart                )
                __field(        __u64,        pa_lstart                )
                __field(        dev_t,        dev                        )
                __field(        __u32,        pa_len                        )

        ),

        TP_fast_assign(
                __entry->dev                = ac->ac_sb->s_dev;
                __entry->ino                = ac->ac_inode->i_ino;
                __entry->pa_pstart        = pa->pa_pstart;
                __entry->pa_lstart        = pa->pa_lstart;
                __entry->pa_len                = pa->pa_len;
        ),

        TP_printk("dev %d,%d ino %llu pstart %llu len %u lstart %llu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->pa_pstart, __entry->pa_len, __entry->pa_lstart)
);

DEFINE_EVENT(ext4__mb_new_pa, ext4_mb_new_inode_pa,

        TP_PROTO(struct ext4_allocation_context *ac,
                 struct ext4_prealloc_space *pa),

        TP_ARGS(ac, pa)
);

DEFINE_EVENT(ext4__mb_new_pa, ext4_mb_new_group_pa,

        TP_PROTO(struct ext4_allocation_context *ac,
                 struct ext4_prealloc_space *pa),

        TP_ARGS(ac, pa)
);

TRACE_EVENT(ext4_mb_release_inode_pa,
        TP_PROTO(struct ext4_prealloc_space *pa,
                 unsigned long long block, unsigned int count),

        TP_ARGS(pa, block, count),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        __u64,        block                        )
                __field(        dev_t,        dev                        )
                __field(        __u32,        count                        )

        ),

        TP_fast_assign(
                __entry->dev                = pa->pa_inode->i_sb->s_dev;
                __entry->ino                = pa->pa_inode->i_ino;
                __entry->block                = block;
                __entry->count                = count;
        ),

        TP_printk("dev %d,%d ino %llu block %llu count %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->block, __entry->count)
);

TRACE_EVENT(ext4_mb_release_group_pa,
        TP_PROTO(struct super_block *sb, struct ext4_prealloc_space *pa),

        TP_ARGS(sb, pa),

        TP_STRUCT__entry(
                __field(        dev_t,        dev                        )
                __field(        __u64,        pa_pstart                )
                __field(        __u32,        pa_len                        )

        ),

        TP_fast_assign(
                __entry->dev                = sb->s_dev;
                __entry->pa_pstart        = pa->pa_pstart;
                __entry->pa_len                = pa->pa_len;
        ),

        TP_printk("dev %d,%d pstart %llu len %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->pa_pstart, __entry->pa_len)
);

TRACE_EVENT(ext4_discard_preallocations,
        TP_PROTO(struct inode *inode, unsigned int len),

        TP_ARGS(inode, len),

        TP_STRUCT__entry(
                __field(        u64,                ino                )
                __field(        dev_t,                dev                )
                __field(        unsigned int,        len                )

        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->len        = len;
        ),

        TP_printk("dev %d,%d ino %llu len: %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino, __entry->len)
);

TRACE_EVENT(ext4_mb_discard_preallocations,
        TP_PROTO(struct super_block *sb, int needed),

        TP_ARGS(sb, needed),

        TP_STRUCT__entry(
                __field(        dev_t,        dev                        )
                __field(        int,        needed                        )

        ),

        TP_fast_assign(
                __entry->dev        = sb->s_dev;
                __entry->needed        = needed;
        ),

        TP_printk("dev %d,%d needed %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->needed)
);

TRACE_EVENT(ext4_request_blocks,
        TP_PROTO(struct ext4_allocation_request *ar),

        TP_ARGS(ar),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        __u64,        goal                        )
                __field(        __u64,        pleft                        )
                __field(        __u64,        pright                        )
                __field(        dev_t,        dev                        )
                __field(        unsigned int, len                )
                __field(        __u32,  logical                        )
                __field(        __u32,        lleft                        )
                __field(        __u32,        lright                        )
                __field(        unsigned int, flags                )
        ),

        TP_fast_assign(
                __entry->dev        = ar->inode->i_sb->s_dev;
                __entry->ino        = ar->inode->i_ino;
                __entry->len        = ar->len;
                __entry->logical = ar->logical;
                __entry->goal        = ar->goal;
                __entry->lleft        = ar->lleft;
                __entry->lright        = ar->lright;
                __entry->pleft        = ar->pleft;
                __entry->pright        = ar->pright;
                __entry->flags        = ar->flags;
        ),

        TP_printk("dev %d,%d ino %llu flags %s len %u lblk %u goal %llu "
                  "lleft %u lright %u pleft %llu pright %llu ",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino, show_mballoc_flags(__entry->flags),
                  __entry->len, __entry->logical, __entry->goal,
                  __entry->lleft, __entry->lright, __entry->pleft,
                  __entry->pright)
);

TRACE_EVENT(ext4_allocate_blocks,
        TP_PROTO(struct ext4_allocation_request *ar, unsigned long long block),

        TP_ARGS(ar, block),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        __u64,        block                        )
                __field(        __u64,        goal                        )
                __field(        __u64,        pleft                        )
                __field(        __u64,        pright                        )
                __field(        dev_t,        dev                        )
                __field(        unsigned int, len                )
                __field(        __u32,  logical                        )
                __field(        __u32,        lleft                        )
                __field(        __u32,        lright                        )
                __field(        unsigned int, flags                )
        ),

        TP_fast_assign(
                __entry->dev        = ar->inode->i_sb->s_dev;
                __entry->ino        = ar->inode->i_ino;
                __entry->block        = block;
                __entry->len        = ar->len;
                __entry->logical = ar->logical;
                __entry->goal        = ar->goal;
                __entry->lleft        = ar->lleft;
                __entry->lright        = ar->lright;
                __entry->pleft        = ar->pleft;
                __entry->pright        = ar->pright;
                __entry->flags        = ar->flags;
        ),

        TP_printk("dev %d,%d ino %llu flags %s len %u block %llu lblk %u "
                  "goal %llu lleft %u lright %u pleft %llu pright %llu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino, show_mballoc_flags(__entry->flags),
                  __entry->len, __entry->block, __entry->logical,
                  __entry->goal,  __entry->lleft, __entry->lright,
                  __entry->pleft, __entry->pright)
);

TRACE_EVENT(ext4_free_blocks,
        TP_PROTO(struct inode *inode, __u64 block, unsigned long count,
                 int flags),

        TP_ARGS(inode, block, count, flags),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        __u64,        block                        )
                __field(        unsigned long,        count                )
                __field(        dev_t,        dev                        )
                __field(        int,        flags                        )
                __field(        __u16,        mode                        )
        ),

        TP_fast_assign(
                __entry->dev                = inode->i_sb->s_dev;
                __entry->ino                = inode->i_ino;
                __entry->block                = block;
                __entry->count                = count;
                __entry->flags                = flags;
                __entry->mode                = inode->i_mode;
        ),

        TP_printk("dev %d,%d ino %llu mode 0%o block %llu count %lu flags %s",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->mode, __entry->block, __entry->count,
                  show_free_flags(__entry->flags))
);

TRACE_EVENT(ext4_sync_file_enter,
        TP_PROTO(struct file *file, int datasync),

        TP_ARGS(file, datasync),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        u64,        parent                        )
                __field(        dev_t,        dev                        )
                __field(        int,        datasync                )
        ),

        TP_fast_assign(
                struct dentry *dentry = file->f_path.dentry;

                __entry->dev                = dentry->d_sb->s_dev;
                __entry->ino                = d_inode(dentry)->i_ino;
                __entry->datasync        = datasync;
                __entry->parent                = d_inode(dentry->d_parent)->i_ino;
        ),

        TP_printk("dev %d,%d ino %llu parent %llu datasync %d ",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->parent, __entry->datasync)
);

TRACE_EVENT(ext4_sync_file_exit,
        TP_PROTO(struct inode *inode, int ret),

        TP_ARGS(inode, ret),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        dev_t,        dev                        )
                __field(        int,        ret                        )
        ),

        TP_fast_assign(
                __entry->dev                = inode->i_sb->s_dev;
                __entry->ino                = inode->i_ino;
                __entry->ret                = ret;
        ),

        TP_printk("dev %d,%d ino %llu ret %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->ret)
);

TRACE_EVENT(ext4_sync_fs,
        TP_PROTO(struct super_block *sb, int wait),

        TP_ARGS(sb, wait),

        TP_STRUCT__entry(
                __field(        dev_t,        dev                        )
                __field(        int,        wait                        )

        ),

        TP_fast_assign(
                __entry->dev        = sb->s_dev;
                __entry->wait        = wait;
        ),

        TP_printk("dev %d,%d wait %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->wait)
);

TRACE_EVENT(ext4_alloc_da_blocks,
        TP_PROTO(struct inode *inode),

        TP_ARGS(inode),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        dev_t,        dev                        )
                __field( unsigned int,        data_blocks                )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->data_blocks = EXT4_I(inode)->i_reserved_data_blocks;
        ),

        TP_printk("dev %d,%d ino %llu reserved_data_blocks %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->data_blocks)
);

TRACE_EVENT(ext4_mballoc_alloc,
        TP_PROTO(struct ext4_allocation_context *ac),

        TP_ARGS(ac),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        dev_t,        dev                        )
                __field(        __u32,         orig_logical                )
                __field(          int,        orig_start                )
                __field(        __u32,         orig_group                )
                __field(          int,        orig_len                )
                __field(        __u32,         goal_logical                )
                __field(          int,        goal_start                )
                __field(        __u32,         goal_group                )
                __field(          int,        goal_len                )
                __field(        __u32,         result_logical                )
                __field(          int,        result_start                )
                __field(        __u32,         result_group                )
                __field(          int,        result_len                )
                __field(        __u16,        found                        )
                __field(        __u16,        groups                        )
                __field(        __u16,        buddy                        )
                __field(        __u16,        flags                        )
                __field(        __u16,        tail                        )
                __field(        __u8,        cr                        )
        ),

        TP_fast_assign(
                __entry->dev                = ac->ac_inode->i_sb->s_dev;
                __entry->ino                = ac->ac_inode->i_ino;
                __entry->orig_logical        = ac->ac_o_ex.fe_logical;
                __entry->orig_start        = ac->ac_o_ex.fe_start;
                __entry->orig_group        = ac->ac_o_ex.fe_group;
                __entry->orig_len        = ac->ac_o_ex.fe_len;
                __entry->goal_logical        = ac->ac_g_ex.fe_logical;
                __entry->goal_start        = ac->ac_g_ex.fe_start;
                __entry->goal_group        = ac->ac_g_ex.fe_group;
                __entry->goal_len        = ac->ac_g_ex.fe_len;
                __entry->result_logical        = ac->ac_f_ex.fe_logical;
                __entry->result_start        = ac->ac_f_ex.fe_start;
                __entry->result_group        = ac->ac_f_ex.fe_group;
                __entry->result_len        = ac->ac_f_ex.fe_len;
                __entry->found                = ac->ac_found;
                __entry->flags                = ac->ac_flags;
                __entry->groups                = ac->ac_groups_scanned;
                __entry->buddy                = ac->ac_buddy;
                __entry->tail                = ac->ac_tail;
                __entry->cr                = ac->ac_criteria;
        ),

        TP_printk("dev %d,%d inode %llu orig %u/%d/%u@%u goal %u/%d/%u@%u "
                  "result %u/%d/%u@%u blks %u grps %u cr %s flags %s "
                  "tail %u broken %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->orig_group, __entry->orig_start,
                  __entry->orig_len, __entry->orig_logical,
                  __entry->goal_group, __entry->goal_start,
                  __entry->goal_len, __entry->goal_logical,
                  __entry->result_group, __entry->result_start,
                  __entry->result_len, __entry->result_logical,
                  __entry->found, __entry->groups, show_criteria(__entry->cr),
                  show_mballoc_flags(__entry->flags), __entry->tail,
                  __entry->buddy ? 1 << __entry->buddy : 0)
);

TRACE_EVENT(ext4_mballoc_prealloc,
        TP_PROTO(struct ext4_allocation_context *ac),

        TP_ARGS(ac),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        dev_t,        dev                        )
                __field(        __u32,         orig_logical                )
                __field(          int,        orig_start                )
                __field(        __u32,         orig_group                )
                __field(          int,        orig_len                )
                __field(        __u32,         result_logical                )
                __field(          int,        result_start                )
                __field(        __u32,         result_group                )
                __field(          int,        result_len                )
        ),

        TP_fast_assign(
                __entry->dev                = ac->ac_inode->i_sb->s_dev;
                __entry->ino                = ac->ac_inode->i_ino;
                __entry->orig_logical        = ac->ac_o_ex.fe_logical;
                __entry->orig_start        = ac->ac_o_ex.fe_start;
                __entry->orig_group        = ac->ac_o_ex.fe_group;
                __entry->orig_len        = ac->ac_o_ex.fe_len;
                __entry->result_logical        = ac->ac_b_ex.fe_logical;
                __entry->result_start        = ac->ac_b_ex.fe_start;
                __entry->result_group        = ac->ac_b_ex.fe_group;
                __entry->result_len        = ac->ac_b_ex.fe_len;
        ),

        TP_printk("dev %d,%d inode %llu orig %u/%d/%u@%u result %u/%d/%u@%u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->orig_group, __entry->orig_start,
                  __entry->orig_len, __entry->orig_logical,
                  __entry->result_group, __entry->result_start,
                  __entry->result_len, __entry->result_logical)
);

DECLARE_EVENT_CLASS(ext4__mballoc,
        TP_PROTO(struct super_block *sb,
                 struct inode *inode,
                 ext4_group_t group,
                 ext4_grpblk_t start,
                 ext4_grpblk_t len),

        TP_ARGS(sb, inode, group, start, len),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        dev_t,        dev                        )
                __field(          int,        result_start                )
                __field(        __u32,         result_group                )
                __field(          int,        result_len                )
        ),

        TP_fast_assign(
                __entry->dev                = sb->s_dev;
                __entry->ino                = inode ? inode->i_ino : 0;
                __entry->result_start        = start;
                __entry->result_group        = group;
                __entry->result_len        = len;
        ),

        TP_printk("dev %d,%d inode %llu extent %u/%d/%d ",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->result_group, __entry->result_start,
                  __entry->result_len)
);

DEFINE_EVENT(ext4__mballoc, ext4_mballoc_discard,

        TP_PROTO(struct super_block *sb,
                 struct inode *inode,
                 ext4_group_t group,
                 ext4_grpblk_t start,
                 ext4_grpblk_t len),

        TP_ARGS(sb, inode, group, start, len)
);

DEFINE_EVENT(ext4__mballoc, ext4_mballoc_free,

        TP_PROTO(struct super_block *sb,
                 struct inode *inode,
                 ext4_group_t group,
                 ext4_grpblk_t start,
                 ext4_grpblk_t len),

        TP_ARGS(sb, inode, group, start, len)
);

TRACE_EVENT(ext4_forget,
        TP_PROTO(struct inode *inode, int is_metadata, __u64 block),

        TP_ARGS(inode, is_metadata, block),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        __u64,        block                        )
                __field(        dev_t,        dev                        )
                __field(        int,        is_metadata                )
                __field(        __u16,        mode                        )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->block        = block;
                __entry->is_metadata = is_metadata;
                __entry->mode        = inode->i_mode;
        ),

        TP_printk("dev %d,%d ino %llu mode 0%o is_metadata %d block %llu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->mode, __entry->is_metadata, __entry->block)
);

TRACE_EVENT(ext4_da_update_reserve_space,
        TP_PROTO(struct inode *inode, int used_blocks, int quota_claim),

        TP_ARGS(inode, used_blocks, quota_claim),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        __u64,        i_blocks                )
                __field(        dev_t,        dev                        )
                __field(        int,        used_blocks                )
                __field(        int,        reserved_data_blocks        )
                __field(        int,        quota_claim                )
                __field(        __u16,        mode                        )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->i_blocks = inode->i_blocks;
                __entry->used_blocks = used_blocks;
                __entry->reserved_data_blocks =
                                EXT4_I(inode)->i_reserved_data_blocks;
                __entry->quota_claim = quota_claim;
                __entry->mode        = inode->i_mode;
        ),

        TP_printk("dev %d,%d ino %llu mode 0%o i_blocks %llu used_blocks %d "
                  "reserved_data_blocks %d quota_claim %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->mode, __entry->i_blocks,
                  __entry->used_blocks, __entry->reserved_data_blocks,
                  __entry->quota_claim)
);

TRACE_EVENT(ext4_da_reserve_space,
        TP_PROTO(struct inode *inode, int nr_resv),

        TP_ARGS(inode, nr_resv),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        __u64,        i_blocks                )
                __field(        dev_t,        dev                        )
                __field(        int,        reserve_blocks                )
                __field(        int,        reserved_data_blocks        )
                __field(        __u16,  mode                        )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->i_blocks = inode->i_blocks;
                __entry->reserve_blocks = nr_resv;
                __entry->reserved_data_blocks = EXT4_I(inode)->i_reserved_data_blocks;
                __entry->mode        = inode->i_mode;
        ),

        TP_printk("dev %d,%d ino %llu mode 0%o i_blocks %llu reserve_blocks %d"
                  "reserved_data_blocks %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->mode, __entry->i_blocks,
                  __entry->reserve_blocks, __entry->reserved_data_blocks)
);

TRACE_EVENT(ext4_da_release_space,
        TP_PROTO(struct inode *inode, int freed_blocks),

        TP_ARGS(inode, freed_blocks),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        __u64,        i_blocks                )
                __field(        dev_t,        dev                        )
                __field(        int,        freed_blocks                )
                __field(        int,        reserved_data_blocks        )
                __field(        __u16,  mode                        )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->i_blocks = inode->i_blocks;
                __entry->freed_blocks = freed_blocks;
                __entry->reserved_data_blocks = EXT4_I(inode)->i_reserved_data_blocks;
                __entry->mode        = inode->i_mode;
        ),

        TP_printk("dev %d,%d ino %llu mode 0%o i_blocks %llu freed_blocks %d "
                  "reserved_data_blocks %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->mode, __entry->i_blocks,
                  __entry->freed_blocks, __entry->reserved_data_blocks)
);

DECLARE_EVENT_CLASS(ext4__bitmap_load,
        TP_PROTO(struct super_block *sb, unsigned long group),

        TP_ARGS(sb, group),

        TP_STRUCT__entry(
                __field(        dev_t,        dev                        )
                __field(        __u32,        group                        )

        ),

        TP_fast_assign(
                __entry->dev        = sb->s_dev;
                __entry->group        = group;
        ),

        TP_printk("dev %d,%d group %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->group)
);

DEFINE_EVENT(ext4__bitmap_load, ext4_mb_bitmap_load,

        TP_PROTO(struct super_block *sb, unsigned long group),

        TP_ARGS(sb, group)
);

DEFINE_EVENT(ext4__bitmap_load, ext4_mb_buddy_bitmap_load,

        TP_PROTO(struct super_block *sb, unsigned long group),

        TP_ARGS(sb, group)
);

DEFINE_EVENT(ext4__bitmap_load, ext4_load_inode_bitmap,

        TP_PROTO(struct super_block *sb, unsigned long group),

        TP_ARGS(sb, group)
);

TRACE_EVENT(ext4_read_block_bitmap_load,
        TP_PROTO(struct super_block *sb, unsigned long group, bool prefetch),

        TP_ARGS(sb, group, prefetch),

        TP_STRUCT__entry(
                __field(        dev_t,        dev                        )
                __field(        __u32,        group                        )
                __field(        bool,        prefetch                )

        ),

        TP_fast_assign(
                __entry->dev        = sb->s_dev;
                __entry->group        = group;
                __entry->prefetch = prefetch;
        ),

        TP_printk("dev %d,%d group %u prefetch %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->group, __entry->prefetch)
);

DECLARE_EVENT_CLASS(ext4__fallocate_mode,
        TP_PROTO(struct inode *inode, loff_t offset, loff_t len, int mode),

        TP_ARGS(inode, offset, len, mode),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        loff_t,        offset                        )
                __field(        loff_t, len                        )
                __field(        dev_t,        dev                        )
                __field(        int,        mode                        )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->offset        = offset;
                __entry->len        = len;
                __entry->mode        = mode;
        ),

        TP_printk("dev %d,%d ino %llu offset %lld len %lld mode %s",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->offset, __entry->len,
                  show_falloc_mode(__entry->mode))
);

DEFINE_EVENT(ext4__fallocate_mode, ext4_fallocate_enter,

        TP_PROTO(struct inode *inode, loff_t offset, loff_t len, int mode),

        TP_ARGS(inode, offset, len, mode)
);

DEFINE_EVENT(ext4__fallocate_mode, ext4_punch_hole,

        TP_PROTO(struct inode *inode, loff_t offset, loff_t len, int mode),

        TP_ARGS(inode, offset, len, mode)
);

DEFINE_EVENT(ext4__fallocate_mode, ext4_zero_range,

        TP_PROTO(struct inode *inode, loff_t offset, loff_t len, int mode),

        TP_ARGS(inode, offset, len, mode)
);

TRACE_EVENT(ext4_fallocate_exit,
        TP_PROTO(struct inode *inode, loff_t offset,
                 unsigned int max_blocks, int ret),

        TP_ARGS(inode, offset, max_blocks, ret),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        loff_t,        pos                        )
                __field(        dev_t,        dev                        )
                __field(        unsigned int,        blocks                )
                __field(        int,         ret                        )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->pos        = offset;
                __entry->blocks        = max_blocks;
                __entry->ret        = ret;
        ),

        TP_printk("dev %d,%d ino %llu pos %lld blocks %u ret %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->pos, __entry->blocks,
                  __entry->ret)
);

TRACE_EVENT(ext4_unlink_enter,
        TP_PROTO(struct inode *parent, struct dentry *dentry),

        TP_ARGS(parent, dentry),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        u64,        parent                        )
                __field(        loff_t,        size                        )
                __field(        dev_t,        dev                        )
        ),

        TP_fast_assign(
                __entry->dev                = dentry->d_sb->s_dev;
                __entry->ino                = d_inode(dentry)->i_ino;
                __entry->parent                = parent->i_ino;
                __entry->size                = d_inode(dentry)->i_size;
        ),

        TP_printk("dev %d,%d ino %llu size %lld parent %llu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino, __entry->size,
                  __entry->parent)
);

TRACE_EVENT(ext4_unlink_exit,
        TP_PROTO(struct dentry *dentry, int ret),

        TP_ARGS(dentry, ret),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        dev_t,        dev                        )
                __field(        int,        ret                        )
        ),

        TP_fast_assign(
                __entry->dev                = dentry->d_sb->s_dev;
                __entry->ino                = d_inode(dentry)->i_ino;
                __entry->ret                = ret;
        ),

        TP_printk("dev %d,%d ino %llu ret %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->ret)
);

DECLARE_EVENT_CLASS(ext4__truncate,
        TP_PROTO(struct inode *inode),

        TP_ARGS(inode),

        TP_STRUCT__entry(
                __field(        u64,                ino                )
                __field(        __u64,                blocks                )
                __field(        dev_t,                dev                )
        ),

        TP_fast_assign(
                __entry->dev    = inode->i_sb->s_dev;
                __entry->ino    = inode->i_ino;
                __entry->blocks        = inode->i_blocks;
        ),

        TP_printk("dev %d,%d ino %llu blocks %llu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino, __entry->blocks)
);

DEFINE_EVENT(ext4__truncate, ext4_truncate_enter,

        TP_PROTO(struct inode *inode),

        TP_ARGS(inode)
);

DEFINE_EVENT(ext4__truncate, ext4_truncate_exit,

        TP_PROTO(struct inode *inode),

        TP_ARGS(inode)
);

/* 'ux' is the unwritten extent. */
TRACE_EVENT(ext4_ext_convert_to_initialized_enter,
        TP_PROTO(struct inode *inode, struct ext4_map_blocks *map,
                 struct ext4_extent *ux),

        TP_ARGS(inode, map, ux),

        TP_STRUCT__entry(
                __field(        u64,                ino        )
                __field(        ext4_fsblk_t,        u_pblk        )
                __field(        dev_t,                dev        )
                __field(        ext4_lblk_t,        m_lblk        )
                __field(        unsigned,        m_len        )
                __field(        ext4_lblk_t,        u_lblk        )
                __field(        unsigned,        u_len        )
        ),

        TP_fast_assign(
                __entry->dev                = inode->i_sb->s_dev;
                __entry->ino                = inode->i_ino;
                __entry->m_lblk                = map->m_lblk;
                __entry->m_len                = map->m_len;
                __entry->u_lblk                = le32_to_cpu(ux->ee_block);
                __entry->u_len                = ext4_ext_get_actual_len(ux);
                __entry->u_pblk                = ext4_ext_pblock(ux);
        ),

        TP_printk("dev %d,%d ino %llu m_lblk %u m_len %u u_lblk %u u_len %u "
                  "u_pblk %llu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->m_lblk, __entry->m_len,
                  __entry->u_lblk, __entry->u_len, __entry->u_pblk)
);

/*
 * 'ux' is the unwritten extent.
 * 'ix' is the initialized extent to which blocks are transferred.
 */
TRACE_EVENT(ext4_ext_convert_to_initialized_fastpath,
        TP_PROTO(struct inode *inode, struct ext4_map_blocks *map,
                 struct ext4_extent *ux, struct ext4_extent *ix),

        TP_ARGS(inode, map, ux, ix),

        TP_STRUCT__entry(
                __field(        u64,                ino        )
                __field(        ext4_fsblk_t,        u_pblk        )
                __field(        ext4_fsblk_t,        i_pblk        )
                __field(        dev_t,                dev        )
                __field(        ext4_lblk_t,        m_lblk        )
                __field(        unsigned,        m_len        )
                __field(        ext4_lblk_t,        u_lblk        )
                __field(        unsigned,        u_len        )
                __field(        ext4_lblk_t,        i_lblk        )
                __field(        unsigned,        i_len        )
        ),

        TP_fast_assign(
                __entry->dev                = inode->i_sb->s_dev;
                __entry->ino                = inode->i_ino;
                __entry->m_lblk                = map->m_lblk;
                __entry->m_len                = map->m_len;
                __entry->u_lblk                = le32_to_cpu(ux->ee_block);
                __entry->u_len                = ext4_ext_get_actual_len(ux);
                __entry->u_pblk                = ext4_ext_pblock(ux);
                __entry->i_lblk                = le32_to_cpu(ix->ee_block);
                __entry->i_len                = ext4_ext_get_actual_len(ix);
                __entry->i_pblk                = ext4_ext_pblock(ix);
        ),

        TP_printk("dev %d,%d ino %llu m_lblk %u m_len %u "
                  "u_lblk %u u_len %u u_pblk %llu "
                  "i_lblk %u i_len %u i_pblk %llu ",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->m_lblk, __entry->m_len,
                  __entry->u_lblk, __entry->u_len, __entry->u_pblk,
                  __entry->i_lblk, __entry->i_len, __entry->i_pblk)
);

DECLARE_EVENT_CLASS(ext4__map_blocks_enter,
        TP_PROTO(struct inode *inode, ext4_lblk_t lblk,
                 unsigned int len, unsigned int flags),

        TP_ARGS(inode, lblk, len, flags),

        TP_STRUCT__entry(
                __field(        u64,                ino                )
                __field(        dev_t,                dev                )
                __field(        ext4_lblk_t,        lblk                )
                __field(        unsigned int,        len                )
                __field(        unsigned int,        flags                )
        ),

        TP_fast_assign(
                __entry->dev    = inode->i_sb->s_dev;
                __entry->ino    = inode->i_ino;
                __entry->lblk        = lblk;
                __entry->len        = len;
                __entry->flags        = flags;
        ),

        TP_printk("dev %d,%d ino %llu lblk %u len %u flags %s",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->lblk, __entry->len, show_map_flags(__entry->flags))
);

DEFINE_EVENT(ext4__map_blocks_enter, ext4_ext_map_blocks_enter,
        TP_PROTO(struct inode *inode, ext4_lblk_t lblk,
                 unsigned len, unsigned flags),

        TP_ARGS(inode, lblk, len, flags)
);

DEFINE_EVENT(ext4__map_blocks_enter, ext4_ind_map_blocks_enter,
        TP_PROTO(struct inode *inode, ext4_lblk_t lblk,
                 unsigned len, unsigned flags),

        TP_ARGS(inode, lblk, len, flags)
);

DECLARE_EVENT_CLASS(ext4__map_blocks_exit,
        TP_PROTO(struct inode *inode, unsigned flags, struct ext4_map_blocks *map,
                 int ret),

        TP_ARGS(inode, flags, map, ret),

        TP_STRUCT__entry(
                __field(        u64,                ino                )
                __field(        ext4_fsblk_t,        pblk                )
                __field(        dev_t,                dev                )
                __field(        unsigned int,        flags                )
                __field(        ext4_lblk_t,        lblk                )
                __field(        unsigned int,        len                )
                __field(        unsigned int,        mflags                )
                __field(        int,                ret                )
        ),

        TP_fast_assign(
                __entry->dev    = inode->i_sb->s_dev;
                __entry->ino    = inode->i_ino;
                __entry->flags        = flags;
                __entry->pblk        = map->m_pblk;
                __entry->lblk        = map->m_lblk;
                __entry->len        = map->m_len;
                __entry->mflags        = map->m_flags;
                __entry->ret        = ret;
        ),

        TP_printk("dev %d,%d ino %llu flags %s lblk %u pblk %llu len %u "
                  "mflags %s ret %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  show_map_flags(__entry->flags), __entry->lblk, __entry->pblk,
                  __entry->len, show_mflags(__entry->mflags), __entry->ret)
);

DEFINE_EVENT(ext4__map_blocks_exit, ext4_ext_map_blocks_exit,
        TP_PROTO(struct inode *inode, unsigned flags,
                 struct ext4_map_blocks *map, int ret),

        TP_ARGS(inode, flags, map, ret)
);

DEFINE_EVENT(ext4__map_blocks_exit, ext4_ind_map_blocks_exit,
        TP_PROTO(struct inode *inode, unsigned flags,
                 struct ext4_map_blocks *map, int ret),

        TP_ARGS(inode, flags, map, ret)
);

TRACE_EVENT(ext4_ext_load_extent,
        TP_PROTO(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk),

        TP_ARGS(inode, lblk, pblk),

        TP_STRUCT__entry(
                __field(        u64,                ino                )
                __field(        ext4_fsblk_t,        pblk                )
                __field(        dev_t,                dev                )
                __field(        ext4_lblk_t,        lblk                )
        ),

        TP_fast_assign(
                __entry->dev    = inode->i_sb->s_dev;
                __entry->ino    = inode->i_ino;
                __entry->pblk        = pblk;
                __entry->lblk        = lblk;
        ),

        TP_printk("dev %d,%d ino %llu lblk %u pblk %llu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->lblk, __entry->pblk)
);

TRACE_EVENT(ext4_load_inode,
        TP_PROTO(struct super_block *sb, unsigned long ino),

        TP_ARGS(sb, ino),

        TP_STRUCT__entry(
                __field(        u64,        ino                )
                __field(        dev_t,        dev                )
        ),

        TP_fast_assign(
                __entry->dev                = sb->s_dev;
                __entry->ino                = ino;
        ),

        TP_printk("dev %d,%d ino %llu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino)
);

TRACE_EVENT(ext4_journal_start_sb,
        TP_PROTO(struct super_block *sb, int blocks, int rsv_blocks,
                 int revoke_creds, int type, unsigned long IP),

        TP_ARGS(sb, blocks, rsv_blocks, revoke_creds, type, IP),

        TP_STRUCT__entry(
                __field(        dev_t,                dev                )
                __field(        unsigned long,        ip                )
                __field(        int,                blocks                )
                __field(        int,                rsv_blocks        )
                __field(        int,                revoke_creds        )
                __field(        int,                type                )
        ),

        TP_fast_assign(
                __entry->dev                 = sb->s_dev;
                __entry->ip                 = IP;
                __entry->blocks                 = blocks;
                __entry->rsv_blocks         = rsv_blocks;
                __entry->revoke_creds         = revoke_creds;
                __entry->type                 = type;
        ),

        TP_printk("dev %d,%d blocks %d, rsv_blocks %d, revoke_creds %d,"
                  " type %d, caller %pS", MAJOR(__entry->dev),
                  MINOR(__entry->dev), __entry->blocks, __entry->rsv_blocks,
                  __entry->revoke_creds, __entry->type, (void *)__entry->ip)
);

TRACE_EVENT(ext4_journal_start_inode,
        TP_PROTO(struct inode *inode, int blocks, int rsv_blocks,
                 int revoke_creds, int type, unsigned long IP),

        TP_ARGS(inode, blocks, rsv_blocks, revoke_creds, type, IP),

        TP_STRUCT__entry(
                __field(        u64,                ino                )
                __field(        dev_t,                dev                )
                __field(        unsigned long,        ip                )
                __field(        int,                blocks                )
                __field(        int,                rsv_blocks        )
                __field(        int,                revoke_creds        )
                __field(        int,                type                )
        ),

        TP_fast_assign(
                __entry->dev                 = inode->i_sb->s_dev;
                __entry->ip                 = IP;
                __entry->blocks                 = blocks;
                __entry->rsv_blocks         = rsv_blocks;
                __entry->revoke_creds         = revoke_creds;
                __entry->type                 = type;
                __entry->ino                 = inode->i_ino;
        ),

        TP_printk("dev %d,%d blocks %d, rsv_blocks %d, revoke_creds %d,"
                  " type %d, ino %llu, caller %pS", MAJOR(__entry->dev),
                  MINOR(__entry->dev), __entry->blocks, __entry->rsv_blocks,
                  __entry->revoke_creds, __entry->type,
                  __entry->ino, (void *)__entry->ip)
);

TRACE_EVENT(ext4_journal_start_reserved,
        TP_PROTO(struct super_block *sb, int blocks, unsigned long IP),

        TP_ARGS(sb, blocks, IP),

        TP_STRUCT__entry(
                __field(        dev_t,        dev                        )
                __field(unsigned long,        ip                        )
                __field(          int,        blocks                        )
        ),

        TP_fast_assign(
                __entry->dev                 = sb->s_dev;
                __entry->ip                 = IP;
                __entry->blocks                 = blocks;
        ),

        TP_printk("dev %d,%d blocks, %d caller %pS",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->blocks, (void *)__entry->ip)
);

DECLARE_EVENT_CLASS(ext4__trim,
        TP_PROTO(struct super_block *sb,
                 ext4_group_t group,
                 ext4_grpblk_t start,
                 ext4_grpblk_t len),

        TP_ARGS(sb, group, start, len),

        TP_STRUCT__entry(
                __field(        int,        dev_major                )
                __field(        int,        dev_minor                )
                __field(        __u32,         group                        )
                __field(        int,        start                        )
                __field(        int,        len                        )
        ),

        TP_fast_assign(
                __entry->dev_major        = MAJOR(sb->s_dev);
                __entry->dev_minor        = MINOR(sb->s_dev);
                __entry->group                = group;
                __entry->start                = start;
                __entry->len                = len;
        ),

        TP_printk("dev %d,%d group %u, start %d, len %d",
                  __entry->dev_major, __entry->dev_minor,
                  __entry->group, __entry->start, __entry->len)
);

DEFINE_EVENT(ext4__trim, ext4_trim_extent,

        TP_PROTO(struct super_block *sb,
                 ext4_group_t group,
                 ext4_grpblk_t start,
                 ext4_grpblk_t len),

        TP_ARGS(sb, group, start, len)
);

DEFINE_EVENT(ext4__trim, ext4_trim_all_free,

        TP_PROTO(struct super_block *sb,
                 ext4_group_t group,
                 ext4_grpblk_t start,
                 ext4_grpblk_t len),

        TP_ARGS(sb, group, start, len)
);

TRACE_EVENT(ext4_ext_handle_unwritten_extents,
        TP_PROTO(struct inode *inode, struct ext4_map_blocks *map, int flags,
                 unsigned int allocated, ext4_fsblk_t newblock),

        TP_ARGS(inode, map, flags, allocated, newblock),

        TP_STRUCT__entry(
                __field(        u64,                ino                )
                __field(        ext4_fsblk_t,        pblk                )
                __field(        ext4_fsblk_t,        newblk                )
                __field(        dev_t,                dev                )
                __field(        int,                flags                )
                __field(        ext4_lblk_t,        lblk                )
                __field(        unsigned int,        len                )
                __field(        unsigned int,        allocated        )
        ),

        TP_fast_assign(
                __entry->dev                = inode->i_sb->s_dev;
                __entry->ino                = inode->i_ino;
                __entry->flags                = flags;
                __entry->lblk                = map->m_lblk;
                __entry->pblk                = map->m_pblk;
                __entry->len                = map->m_len;
                __entry->allocated        = allocated;
                __entry->newblk                = newblock;
        ),

        TP_printk("dev %d,%d ino %llu m_lblk %u m_pblk %llu m_len %u flags %s "
                  "allocated %d newblock %llu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  (unsigned) __entry->lblk, (unsigned long long) __entry->pblk,
                  __entry->len, show_map_flags(__entry->flags),
                  (unsigned int) __entry->allocated,
                  (unsigned long long) __entry->newblk)
);

TRACE_EVENT(ext4_get_implied_cluster_alloc_exit,
        TP_PROTO(struct super_block *sb, struct ext4_map_blocks *map, int ret),

        TP_ARGS(sb, map, ret),

        TP_STRUCT__entry(
                __field(        dev_t,                dev        )
                __field(        unsigned int,        flags        )
                __field(        ext4_lblk_t,        lblk        )
                __field(        ext4_fsblk_t,        pblk        )
                __field(        unsigned int,        len        )
                __field(        int,                ret        )
        ),

        TP_fast_assign(
                __entry->dev        = sb->s_dev;
                __entry->flags        = map->m_flags;
                __entry->lblk        = map->m_lblk;
                __entry->pblk        = map->m_pblk;
                __entry->len        = map->m_len;
                __entry->ret        = ret;
        ),

        TP_printk("dev %d,%d m_lblk %u m_pblk %llu m_len %u m_flags %s ret %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->lblk, (unsigned long long) __entry->pblk,
                  __entry->len, show_mflags(__entry->flags), __entry->ret)
);

TRACE_EVENT(ext4_ext_show_extent,
        TP_PROTO(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk,
                 unsigned short len),

        TP_ARGS(inode, lblk, pblk, len),

        TP_STRUCT__entry(
                __field(        u64,                ino        )
                __field(        ext4_fsblk_t,        pblk        )
                __field(        dev_t,                dev        )
                __field(        ext4_lblk_t,        lblk        )
                __field(        unsigned short,        len        )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->pblk        = pblk;
                __entry->lblk        = lblk;
                __entry->len        = len;
        ),

        TP_printk("dev %d,%d ino %llu lblk %u pblk %llu len %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  (unsigned) __entry->lblk,
                  (unsigned long long) __entry->pblk,
                  (unsigned short) __entry->len)
);

TRACE_EVENT(ext4_remove_blocks,
        TP_PROTO(struct inode *inode, struct ext4_extent *ex,
                 ext4_lblk_t from, ext4_fsblk_t to,
                 struct partial_cluster *pc),

        TP_ARGS(inode, ex, from, to, pc),

        TP_STRUCT__entry(
                __field(        u64,                ino        )
                __field(        ext4_fsblk_t,        ee_pblk        )
                __field(        ext4_fsblk_t,        pc_pclu        )
                __field(        dev_t,                dev        )
                __field(        ext4_lblk_t,        from        )
                __field(        ext4_lblk_t,        to        )
                __field(        ext4_lblk_t,        ee_lblk        )
                __field(        unsigned short,        ee_len        )
                __field(        ext4_lblk_t,        pc_lblk        )
                __field(        int,                pc_state)
        ),

        TP_fast_assign(
                __entry->dev                = inode->i_sb->s_dev;
                __entry->ino                = inode->i_ino;
                __entry->from                = from;
                __entry->to                = to;
                __entry->ee_pblk        = ext4_ext_pblock(ex);
                __entry->ee_lblk        = le32_to_cpu(ex->ee_block);
                __entry->ee_len                = ext4_ext_get_actual_len(ex);
                __entry->pc_pclu        = pc->pclu;
                __entry->pc_lblk        = pc->lblk;
                __entry->pc_state        = pc->state;
        ),

        TP_printk("dev %d,%d ino %llu extent [%u(%llu), %u]"
                  "from %u to %u partial [pclu %lld lblk %u state %d]",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  (unsigned) __entry->ee_lblk,
                  (unsigned long long) __entry->ee_pblk,
                  (unsigned short) __entry->ee_len,
                  (unsigned) __entry->from,
                  (unsigned) __entry->to,
                  (long long) __entry->pc_pclu,
                  (unsigned int) __entry->pc_lblk,
                  (int) __entry->pc_state)
);

TRACE_EVENT(ext4_ext_rm_leaf,
        TP_PROTO(struct inode *inode, ext4_lblk_t start,
                 struct ext4_extent *ex,
                 struct partial_cluster *pc),

        TP_ARGS(inode, start, ex, pc),

        TP_STRUCT__entry(
                __field(        u64,                ino        )
                __field(        ext4_fsblk_t,        ee_pblk        )
                __field(        ext4_fsblk_t,        pc_pclu        )
                __field(        dev_t,                dev        )
                __field(        ext4_lblk_t,        start        )
                __field(        ext4_lblk_t,        ee_lblk        )
                __field(        short,                ee_len        )
                __field(        ext4_lblk_t,        pc_lblk        )
                __field(        int,                pc_state)
        ),

        TP_fast_assign(
                __entry->dev                = inode->i_sb->s_dev;
                __entry->ino                = inode->i_ino;
                __entry->start                = start;
                __entry->ee_lblk        = le32_to_cpu(ex->ee_block);
                __entry->ee_pblk        = ext4_ext_pblock(ex);
                __entry->ee_len                = ext4_ext_get_actual_len(ex);
                __entry->pc_pclu        = pc->pclu;
                __entry->pc_lblk        = pc->lblk;
                __entry->pc_state        = pc->state;
        ),

        TP_printk("dev %d,%d ino %llu start_lblk %u last_extent [%u(%llu), %u]"
                  "partial [pclu %lld lblk %u state %d]",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  (unsigned) __entry->start,
                  (unsigned) __entry->ee_lblk,
                  (unsigned long long) __entry->ee_pblk,
                  (unsigned short) __entry->ee_len,
                  (long long) __entry->pc_pclu,
                  (unsigned int) __entry->pc_lblk,
                  (int) __entry->pc_state)
);

TRACE_EVENT(ext4_ext_rm_idx,
        TP_PROTO(struct inode *inode, ext4_fsblk_t pblk),

        TP_ARGS(inode, pblk),

        TP_STRUCT__entry(
                __field(        u64,                ino        )
                __field(        ext4_fsblk_t,        pblk        )
                __field(        dev_t,                dev        )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->pblk        = pblk;
        ),

        TP_printk("dev %d,%d ino %llu index_pblk %llu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  (unsigned long long) __entry->pblk)
);

TRACE_EVENT(ext4_ext_remove_space,
        TP_PROTO(struct inode *inode, ext4_lblk_t start,
                 ext4_lblk_t end, int depth),

        TP_ARGS(inode, start, end, depth),

        TP_STRUCT__entry(
                __field(        u64,                ino        )
                __field(        dev_t,                dev        )
                __field(        ext4_lblk_t,        start        )
                __field(        ext4_lblk_t,        end        )
                __field(        int,                depth        )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->start        = start;
                __entry->end        = end;
                __entry->depth        = depth;
        ),

        TP_printk("dev %d,%d ino %llu since %u end %u depth %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  (unsigned) __entry->start,
                  (unsigned) __entry->end,
                  __entry->depth)
);

TRACE_EVENT(ext4_ext_remove_space_done,
        TP_PROTO(struct inode *inode, ext4_lblk_t start, ext4_lblk_t end,
                 int depth, struct partial_cluster *pc, __le16 eh_entries),

        TP_ARGS(inode, start, end, depth, pc, eh_entries),

        TP_STRUCT__entry(
                __field(        u64,                ino                )
                __field(        ext4_fsblk_t,        pc_pclu                )
                __field(        dev_t,                dev                )
                __field(        ext4_lblk_t,        start                )
                __field(        ext4_lblk_t,        end                )
                __field(        int,                depth                )
                __field(        ext4_lblk_t,        pc_lblk                )
                __field(        int,                pc_state        )
                __field(        unsigned short,        eh_entries        )
        ),

        TP_fast_assign(
                __entry->dev                = inode->i_sb->s_dev;
                __entry->ino                = inode->i_ino;
                __entry->start                = start;
                __entry->end                = end;
                __entry->depth                = depth;
                __entry->pc_pclu        = pc->pclu;
                __entry->pc_lblk        = pc->lblk;
                __entry->pc_state        = pc->state;
                __entry->eh_entries        = le16_to_cpu(eh_entries);
        ),

        TP_printk("dev %d,%d ino %llu since %u end %u depth %d "
                  "partial [pclu %lld lblk %u state %d] "
                  "remaining_entries %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  (unsigned) __entry->start,
                  (unsigned) __entry->end,
                  __entry->depth,
                  (long long) __entry->pc_pclu,
                  (unsigned int) __entry->pc_lblk,
                  (int) __entry->pc_state,
                  (unsigned short) __entry->eh_entries)
);

DECLARE_EVENT_CLASS(ext4__es_extent,
        TP_PROTO(struct inode *inode, struct extent_status *es),

        TP_ARGS(inode, es),

        TP_STRUCT__entry(
                __field(        u64,                ino                )
                __field(        u64,                seq                )
                __field(        ext4_fsblk_t,        pblk                )
                __field(        dev_t,                dev                )
                __field(        ext4_lblk_t,        lblk                )
                __field(        ext4_lblk_t,        len                )
                __field(        char,                status                )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->lblk        = es->es_lblk;
                __entry->len        = es->es_len;
                __entry->pblk        = ext4_es_show_pblock(es);
                __entry->status        = ext4_es_status(es);
                __entry->seq        = EXT4_I(inode)->i_es_seq;
        ),

        TP_printk("dev %d,%d ino %llu es [%u/%u) mapped %llu status %s seq %llu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->lblk, __entry->len,
                  __entry->pblk, show_extent_status(__entry->status),
                  __entry->seq)
);

DEFINE_EVENT(ext4__es_extent, ext4_es_insert_extent,
        TP_PROTO(struct inode *inode, struct extent_status *es),

        TP_ARGS(inode, es)
);

DEFINE_EVENT(ext4__es_extent, ext4_es_cache_extent,
        TP_PROTO(struct inode *inode, struct extent_status *es),

        TP_ARGS(inode, es)
);

TRACE_EVENT(ext4_es_remove_extent,
        TP_PROTO(struct inode *inode, ext4_lblk_t lblk, ext4_lblk_t len),

        TP_ARGS(inode, lblk, len),

        TP_STRUCT__entry(
                __field(        u64,        ino                        )
                __field(        loff_t,        lblk                        )
                __field(        loff_t,        len                        )
                __field(        u64,        seq                        )
                __field(        dev_t,        dev                        )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->lblk        = lblk;
                __entry->len        = len;
                __entry->seq        = EXT4_I(inode)->i_es_seq;
        ),

        TP_printk("dev %d,%d ino %llu es [%lld/%lld) seq %llu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->lblk, __entry->len, __entry->seq)
);

TRACE_EVENT(ext4_es_find_extent_range_enter,
        TP_PROTO(struct inode *inode, ext4_lblk_t lblk),

        TP_ARGS(inode, lblk),

        TP_STRUCT__entry(
                __field(        u64,                ino                )
                __field(        dev_t,                dev                )
                __field(        ext4_lblk_t,        lblk                )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->lblk        = lblk;
        ),

        TP_printk("dev %d,%d ino %llu lblk %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino, __entry->lblk)
);

TRACE_EVENT(ext4_es_find_extent_range_exit,
        TP_PROTO(struct inode *inode, struct extent_status *es),

        TP_ARGS(inode, es),

        TP_STRUCT__entry(
                __field(        u64,                ino                )
                __field(        ext4_fsblk_t,        pblk                )
                __field(        dev_t,                dev                )
                __field(        ext4_lblk_t,        lblk                )
                __field(        ext4_lblk_t,        len                )
                __field(        char, status        )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->lblk        = es->es_lblk;
                __entry->len        = es->es_len;
                __entry->pblk        = ext4_es_show_pblock(es);
                __entry->status        = ext4_es_status(es);
        ),

        TP_printk("dev %d,%d ino %llu es [%u/%u) mapped %llu status %s",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->lblk, __entry->len,
                  __entry->pblk, show_extent_status(__entry->status))
);

TRACE_EVENT(ext4_es_lookup_extent_enter,
        TP_PROTO(struct inode *inode, ext4_lblk_t lblk),

        TP_ARGS(inode, lblk),

        TP_STRUCT__entry(
                __field(        u64,                ino                )
                __field(        dev_t,                dev                )
                __field(        ext4_lblk_t,        lblk                )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->lblk        = lblk;
        ),

        TP_printk("dev %d,%d ino %llu lblk %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino, __entry->lblk)
);

TRACE_EVENT(ext4_es_lookup_extent_exit,
        TP_PROTO(struct inode *inode, struct extent_status *es,
                 int found),

        TP_ARGS(inode, es, found),

        TP_STRUCT__entry(
                __field(        u64,                ino                )
                __field(        ext4_fsblk_t,        pblk                )
                __field(        dev_t,                dev                )
                __field(        ext4_lblk_t,        lblk                )
                __field(        ext4_lblk_t,        len                )
                __field(        int,                found                )
                __field(        char,                status                )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->lblk        = es->es_lblk;
                __entry->len        = es->es_len;
                __entry->pblk        = ext4_es_show_pblock(es);
                __entry->status        = ext4_es_status(es);
                __entry->found        = found;
        ),

        TP_printk("dev %d,%d ino %llu found %d [%u/%u) %llu %s",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino, __entry->found,
                  __entry->lblk, __entry->len,
                  __entry->found ? __entry->pblk : 0,
                  show_extent_status(__entry->found ? __entry->status : 0))
);

DECLARE_EVENT_CLASS(ext4__es_shrink_enter,
        TP_PROTO(struct super_block *sb, int nr_to_scan, int cache_cnt),

        TP_ARGS(sb, nr_to_scan, cache_cnt),

        TP_STRUCT__entry(
                __field(        dev_t,        dev                        )
                __field(        int,        nr_to_scan                )
                __field(        int,        cache_cnt                )
        ),

        TP_fast_assign(
                __entry->dev                = sb->s_dev;
                __entry->nr_to_scan        = nr_to_scan;
                __entry->cache_cnt        = cache_cnt;
        ),

        TP_printk("dev %d,%d nr_to_scan %d cache_cnt %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->nr_to_scan, __entry->cache_cnt)
);

DEFINE_EVENT(ext4__es_shrink_enter, ext4_es_shrink_count,
        TP_PROTO(struct super_block *sb, int nr_to_scan, int cache_cnt),

        TP_ARGS(sb, nr_to_scan, cache_cnt)
);

DEFINE_EVENT(ext4__es_shrink_enter, ext4_es_shrink_scan_enter,
        TP_PROTO(struct super_block *sb, int nr_to_scan, int cache_cnt),

        TP_ARGS(sb, nr_to_scan, cache_cnt)
);

TRACE_EVENT(ext4_es_shrink_scan_exit,
        TP_PROTO(struct super_block *sb, int nr_shrunk, int cache_cnt),

        TP_ARGS(sb, nr_shrunk, cache_cnt),

        TP_STRUCT__entry(
                __field(        dev_t,        dev                        )
                __field(        int,        nr_shrunk                )
                __field(        int,        cache_cnt                )
        ),

        TP_fast_assign(
                __entry->dev                = sb->s_dev;
                __entry->nr_shrunk        = nr_shrunk;
                __entry->cache_cnt        = cache_cnt;
        ),

        TP_printk("dev %d,%d nr_shrunk %d cache_cnt %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->nr_shrunk, __entry->cache_cnt)
);

TRACE_EVENT(ext4_collapse_range,
        TP_PROTO(struct inode *inode, loff_t offset, loff_t len),

        TP_ARGS(inode, offset, len),

        TP_STRUCT__entry(
                __field(u64,        ino)
                __field(loff_t,        offset)
                __field(loff_t, len)
                __field(dev_t,        dev)
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->offset        = offset;
                __entry->len        = len;
        ),

        TP_printk("dev %d,%d ino %llu offset %lld len %lld",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->offset, __entry->len)
);

TRACE_EVENT(ext4_insert_range,
        TP_PROTO(struct inode *inode, loff_t offset, loff_t len),

        TP_ARGS(inode, offset, len),

        TP_STRUCT__entry(
                __field(u64,        ino)
                __field(loff_t,        offset)
                __field(loff_t, len)
                __field(dev_t,        dev)
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->offset        = offset;
                __entry->len        = len;
        ),

        TP_printk("dev %d,%d ino %llu offset %lld len %lld",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->offset, __entry->len)
);

TRACE_EVENT(ext4_es_shrink,
        TP_PROTO(struct super_block *sb, int nr_shrunk, u64 scan_time,
                 int nr_skipped, int retried),

        TP_ARGS(sb, nr_shrunk, scan_time, nr_skipped, retried),

        TP_STRUCT__entry(
                __field(        dev_t,                dev                )
                __field(        int,                nr_shrunk        )
                __field(        unsigned long long, scan_time        )
                __field(        int,                nr_skipped        )
                __field(        int,                retried                )
        ),

        TP_fast_assign(
                __entry->dev                = sb->s_dev;
                __entry->nr_shrunk        = nr_shrunk;
                __entry->scan_time        = div_u64(scan_time, 1000);
                __entry->nr_skipped        = nr_skipped;
                __entry->retried        = retried;
        ),

        TP_printk("dev %d,%d nr_shrunk %d, scan_time %llu "
                  "nr_skipped %d retried %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev), __entry->nr_shrunk,
                  __entry->scan_time, __entry->nr_skipped, __entry->retried)
);

TRACE_EVENT(ext4_es_insert_delayed_extent,
        TP_PROTO(struct inode *inode, struct extent_status *es,
                 bool lclu_allocated, bool end_allocated),

        TP_ARGS(inode, es, lclu_allocated, end_allocated),

        TP_STRUCT__entry(
                __field(        u64,                ino                )
                __field(        u64,                seq                )
                __field(        ext4_fsblk_t,        pblk                )
                __field(        dev_t,                dev                )
                __field(        ext4_lblk_t,        lblk                )
                __field(        ext4_lblk_t,        len                )
                __field(        char,                status                )
                __field(        bool,                lclu_allocated        )
                __field(        bool,                end_allocated        )
        ),

        TP_fast_assign(
                __entry->dev                = inode->i_sb->s_dev;
                __entry->ino                = inode->i_ino;
                __entry->lblk                = es->es_lblk;
                __entry->len                = es->es_len;
                __entry->pblk                = ext4_es_show_pblock(es);
                __entry->status                = ext4_es_status(es);
                __entry->lclu_allocated        = lclu_allocated;
                __entry->end_allocated        = end_allocated;
                __entry->seq                = EXT4_I(inode)->i_es_seq;
        ),

        TP_printk("dev %d,%d ino %llu es [%u/%u) mapped %llu status %s allocated %d %d seq %llu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->ino,
                  __entry->lblk, __entry->len,
                  __entry->pblk, show_extent_status(__entry->status),
                  __entry->lclu_allocated, __entry->end_allocated,
                  __entry->seq)
);

/* fsmap traces */
DECLARE_EVENT_CLASS(ext4_fsmap_class,
        TP_PROTO(struct super_block *sb, u32 keydev, u32 agno, u64 bno, u64 len,
                 u64 owner),
        TP_ARGS(sb, keydev, agno, bno, len, owner),
        TP_STRUCT__entry(
                __field(dev_t, dev)
                __field(dev_t, keydev)
                __field(u32, agno)
                __field(u64, bno)
                __field(u64, len)
                __field(u64, owner)
        ),
        TP_fast_assign(
                __entry->dev = sb->s_bdev->bd_dev;
                __entry->keydev = new_decode_dev(keydev);
                __entry->agno = agno;
                __entry->bno = bno;
                __entry->len = len;
                __entry->owner = owner;
        ),
        TP_printk("dev %d:%d keydev %d:%d agno %u bno %llu len %llu owner %lld\n",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  MAJOR(__entry->keydev), MINOR(__entry->keydev),
                  __entry->agno,
                  __entry->bno,
                  __entry->len,
                  __entry->owner)
)
#define DEFINE_FSMAP_EVENT(name) \
DEFINE_EVENT(ext4_fsmap_class, name, \
        TP_PROTO(struct super_block *sb, u32 keydev, u32 agno, u64 bno, u64 len, \
                 u64 owner), \
        TP_ARGS(sb, keydev, agno, bno, len, owner))
DEFINE_FSMAP_EVENT(ext4_fsmap_low_key);
DEFINE_FSMAP_EVENT(ext4_fsmap_high_key);
DEFINE_FSMAP_EVENT(ext4_fsmap_mapping);

DECLARE_EVENT_CLASS(ext4_getfsmap_class,
        TP_PROTO(struct super_block *sb, struct ext4_fsmap *fsmap),
        TP_ARGS(sb, fsmap),
        TP_STRUCT__entry(
                __field(dev_t, dev)
                __field(dev_t, keydev)
                __field(u64, block)
                __field(u64, len)
                __field(u64, owner)
                __field(u64, flags)
        ),
        TP_fast_assign(
                __entry->dev = sb->s_bdev->bd_dev;
                __entry->keydev = new_decode_dev(fsmap->fmr_device);
                __entry->block = fsmap->fmr_physical;
                __entry->len = fsmap->fmr_length;
                __entry->owner = fsmap->fmr_owner;
                __entry->flags = fsmap->fmr_flags;
        ),
        TP_printk("dev %d:%d keydev %d:%d block %llu len %llu owner %lld flags 0x%llx\n",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  MAJOR(__entry->keydev), MINOR(__entry->keydev),
                  __entry->block,
                  __entry->len,
                  __entry->owner,
                  __entry->flags)
)
#define DEFINE_GETFSMAP_EVENT(name) \
DEFINE_EVENT(ext4_getfsmap_class, name, \
        TP_PROTO(struct super_block *sb, struct ext4_fsmap *fsmap), \
        TP_ARGS(sb, fsmap))
DEFINE_GETFSMAP_EVENT(ext4_getfsmap_low_key);
DEFINE_GETFSMAP_EVENT(ext4_getfsmap_high_key);
DEFINE_GETFSMAP_EVENT(ext4_getfsmap_mapping);

TRACE_EVENT(ext4_shutdown,
        TP_PROTO(struct super_block *sb, unsigned long flags),

        TP_ARGS(sb, flags),

        TP_STRUCT__entry(
                __field(        dev_t,        dev                        )
                __field(     unsigned,        flags                        )
        ),

        TP_fast_assign(
                __entry->dev        = sb->s_dev;
                __entry->flags        = flags;
        ),

        TP_printk("dev %d,%d flags %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->flags)
);

TRACE_EVENT(ext4_error,
        TP_PROTO(struct super_block *sb, const char *function,
                 unsigned int line),

        TP_ARGS(sb, function, line),

        TP_STRUCT__entry(
                __field(        dev_t,        dev                        )
                __field( const char *,        function                )
                __field(     unsigned,        line                        )
        ),

        TP_fast_assign(
                __entry->dev        = sb->s_dev;
                __entry->function = function;
                __entry->line        = line;
        ),

        TP_printk("dev %d,%d function %s line %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->function, __entry->line)
);

TRACE_EVENT(ext4_prefetch_bitmaps,
            TP_PROTO(struct super_block *sb, ext4_group_t group,
                     ext4_group_t next, unsigned int prefetch_ios),

        TP_ARGS(sb, group, next, prefetch_ios),

        TP_STRUCT__entry(
                __field(        dev_t,        dev                        )
                __field(        __u32,        group                        )
                __field(        __u32,        next                        )
                __field(        __u32,        ios                        )
        ),

        TP_fast_assign(
                __entry->dev        = sb->s_dev;
                __entry->group        = group;
                __entry->next        = next;
                __entry->ios        = prefetch_ios;
        ),

        TP_printk("dev %d,%d group %u next %u ios %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->group, __entry->next, __entry->ios)
);

TRACE_EVENT(ext4_lazy_itable_init,
            TP_PROTO(struct super_block *sb, ext4_group_t group),

        TP_ARGS(sb, group),

        TP_STRUCT__entry(
                __field(        dev_t,        dev                        )
                __field(        __u32,        group                        )
        ),

        TP_fast_assign(
                __entry->dev        = sb->s_dev;
                __entry->group        = group;
        ),

        TP_printk("dev %d,%d group %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev), __entry->group)
);

TRACE_EVENT(ext4_fc_replay_scan,
        TP_PROTO(struct super_block *sb, int error, int off),

        TP_ARGS(sb, error, off),

        TP_STRUCT__entry(
                __field(dev_t, dev)
                __field(int, error)
                __field(int, off)
        ),

        TP_fast_assign(
                __entry->dev = sb->s_dev;
                __entry->error = error;
                __entry->off = off;
        ),

        TP_printk("dev %d,%d error %d, off %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->error, __entry->off)
);

TRACE_EVENT(ext4_fc_replay,
        TP_PROTO(struct super_block *sb, int tag, int ino, int priv1, int priv2),

        TP_ARGS(sb, tag, ino, priv1, priv2),

        TP_STRUCT__entry(
                __field(dev_t, dev)
                __field(int, tag)
                __field(int, ino)
                __field(int, priv1)
                __field(int, priv2)
        ),

        TP_fast_assign(
                __entry->dev = sb->s_dev;
                __entry->tag = tag;
                __entry->ino = ino;
                __entry->priv1 = priv1;
                __entry->priv2 = priv2;
        ),

        TP_printk("dev %d,%d: tag %d, ino %d, data1 %d, data2 %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->tag, __entry->ino, __entry->priv1, __entry->priv2)
);

TRACE_EVENT(ext4_fc_commit_start,
        TP_PROTO(struct super_block *sb, tid_t commit_tid),

        TP_ARGS(sb, commit_tid),

        TP_STRUCT__entry(
                __field(dev_t, dev)
                __field(tid_t, tid)
        ),

        TP_fast_assign(
                __entry->dev = sb->s_dev;
                __entry->tid = commit_tid;
        ),

        TP_printk("dev %d,%d tid %u", MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->tid)
);

TRACE_EVENT(ext4_fc_commit_stop,
            TP_PROTO(struct super_block *sb, int nblks, int reason,
                     tid_t commit_tid),

        TP_ARGS(sb, nblks, reason, commit_tid),

        TP_STRUCT__entry(
                __field(dev_t, dev)
                __field(int, nblks)
                __field(int, reason)
                __field(int, num_fc)
                __field(int, num_fc_ineligible)
                __field(int, nblks_agg)
                __field(tid_t, tid)
        ),

        TP_fast_assign(
                __entry->dev = sb->s_dev;
                __entry->nblks = nblks;
                __entry->reason = reason;
                __entry->num_fc = EXT4_SB(sb)->s_fc_stats.fc_num_commits;
                __entry->num_fc_ineligible =
                        EXT4_SB(sb)->s_fc_stats.fc_ineligible_commits;
                __entry->nblks_agg = EXT4_SB(sb)->s_fc_stats.fc_numblks;
                __entry->tid = commit_tid;
        ),

        TP_printk("dev %d,%d nblks %d, reason %d, fc = %d, ineligible = %d, agg_nblks %d, tid %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->nblks, __entry->reason, __entry->num_fc,
                  __entry->num_fc_ineligible, __entry->nblks_agg, __entry->tid)
);

#define EM(a)        { EXT4_FC_SNAP_ERR_##a, #a },
#define EMe(a)        { EXT4_FC_SNAP_ERR_##a, #a }

TRACE_EVENT(ext4_fc_lock_updates,
            TP_PROTO(struct super_block *sb, tid_t commit_tid, u64 locked_ns,
                     unsigned int nr_inodes, unsigned int nr_ranges, int err,
                     int snap_err),

        TP_ARGS(sb, commit_tid, locked_ns, nr_inodes, nr_ranges, err, snap_err),

        TP_STRUCT__entry(/* entry */
                __field(dev_t, dev)
                __field(tid_t, tid)
                __field(u64, locked_ns)
                __field(unsigned int, nr_inodes)
                __field(unsigned int, nr_ranges)
                __field(int, err)
                __field(int, snap_err)
        ),

        TP_fast_assign(/* assign */
                __entry->dev = sb->s_dev;
                __entry->tid = commit_tid;
                __entry->locked_ns = locked_ns;
                __entry->nr_inodes = nr_inodes;
                __entry->nr_ranges = nr_ranges;
                __entry->err = err;
                __entry->snap_err = snap_err;
        ),

        TP_printk("dev %d,%d tid %u locked_ns %llu nr_inodes %u nr_ranges %u err %d snap_err %s",
                  MAJOR(__entry->dev), MINOR(__entry->dev), __entry->tid,
                  __entry->locked_ns, __entry->nr_inodes, __entry->nr_ranges,
                  __entry->err, __print_symbolic(__entry->snap_err,
                                                 TRACE_SNAP_ERR))
);

#undef EM
#undef EMe
#undef TRACE_SNAP_ERR

#define FC_REASON_NAME_STAT(reason)                                        \
        show_fc_reason(reason),                                                \
        __entry->fc_ineligible_rc[reason]

TRACE_EVENT(ext4_fc_stats,
        TP_PROTO(struct super_block *sb),

        TP_ARGS(sb),

        TP_STRUCT__entry(
                __field(dev_t, dev)
                __array(unsigned int, fc_ineligible_rc, EXT4_FC_REASON_MAX)
                __field(unsigned long, fc_commits)
                __field(unsigned long, fc_ineligible_commits)
                __field(unsigned long, fc_numblks)
        ),

        TP_fast_assign(
                int i;

                __entry->dev = sb->s_dev;
                for (i = 0; i < EXT4_FC_REASON_MAX; i++) {
                        __entry->fc_ineligible_rc[i] =
                                EXT4_SB(sb)->s_fc_stats.fc_ineligible_reason_count[i];
                }
                __entry->fc_commits = EXT4_SB(sb)->s_fc_stats.fc_num_commits;
                __entry->fc_ineligible_commits =
                        EXT4_SB(sb)->s_fc_stats.fc_ineligible_commits;
                __entry->fc_numblks = EXT4_SB(sb)->s_fc_stats.fc_numblks;
        ),

        TP_printk("dev %d,%d fc ineligible reasons:\n"
                  "%s:%u, %s:%u, %s:%u, %s:%u, %s:%u, %s:%u, %s:%u, %s:%u, %s:%u, %s:%u"
                  "num_commits:%lu, ineligible: %lu, numblks: %lu",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  FC_REASON_NAME_STAT(EXT4_FC_REASON_XATTR),
                  FC_REASON_NAME_STAT(EXT4_FC_REASON_CROSS_RENAME),
                  FC_REASON_NAME_STAT(EXT4_FC_REASON_JOURNAL_FLAG_CHANGE),
                  FC_REASON_NAME_STAT(EXT4_FC_REASON_NOMEM),
                  FC_REASON_NAME_STAT(EXT4_FC_REASON_SWAP_BOOT),
                  FC_REASON_NAME_STAT(EXT4_FC_REASON_RESIZE),
                  FC_REASON_NAME_STAT(EXT4_FC_REASON_RENAME_DIR),
                  FC_REASON_NAME_STAT(EXT4_FC_REASON_FALLOC_RANGE),
                  FC_REASON_NAME_STAT(EXT4_FC_REASON_INODE_JOURNAL_DATA),
                  FC_REASON_NAME_STAT(EXT4_FC_REASON_ENCRYPTED_FILENAME),
                  __entry->fc_commits, __entry->fc_ineligible_commits,
                  __entry->fc_numblks)
);

DECLARE_EVENT_CLASS(ext4_fc_track_dentry,

        TP_PROTO(handle_t *handle, struct inode *inode,
                 struct dentry *dentry, int ret),

        TP_ARGS(handle, inode, dentry, ret),

        TP_STRUCT__entry(
                __field(u64, i_ino)
                __field(dev_t, dev)
                __field(tid_t, t_tid)
                __field(tid_t, i_sync_tid)
                __field(int, error)
        ),

        TP_fast_assign(
                struct ext4_inode_info *ei = EXT4_I(inode);

                __entry->dev = inode->i_sb->s_dev;
                __entry->t_tid = handle->h_transaction->t_tid;
                __entry->i_ino = inode->i_ino;
                __entry->i_sync_tid = ei->i_sync_tid;
                __entry->error = ret;
        ),

        TP_printk("dev %d,%d, t_tid %u, ino %llu, i_sync_tid %u, error %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->t_tid, __entry->i_ino, __entry->i_sync_tid,
                  __entry->error
        )
);

#define DEFINE_EVENT_CLASS_DENTRY(__type)                                \
DEFINE_EVENT(ext4_fc_track_dentry, ext4_fc_track_##__type,                \
        TP_PROTO(handle_t *handle, struct inode *inode,                        \
                 struct dentry *dentry, int ret),                        \
        TP_ARGS(handle, inode, dentry, ret)                                \
)

DEFINE_EVENT_CLASS_DENTRY(create);
DEFINE_EVENT_CLASS_DENTRY(link);
DEFINE_EVENT_CLASS_DENTRY(unlink);

TRACE_EVENT(ext4_fc_track_inode,
        TP_PROTO(handle_t *handle, struct inode *inode, int ret),

        TP_ARGS(handle, inode, ret),

        TP_STRUCT__entry(
                __field(u64, i_ino)
                __field(dev_t, dev)
                __field(tid_t, t_tid)
                __field(tid_t, i_sync_tid)
                __field(int, error)
        ),

        TP_fast_assign(
                struct ext4_inode_info *ei = EXT4_I(inode);

                __entry->dev = inode->i_sb->s_dev;
                __entry->t_tid = handle->h_transaction->t_tid;
                __entry->i_ino = inode->i_ino;
                __entry->i_sync_tid = ei->i_sync_tid;
                __entry->error = ret;
        ),

        TP_printk("dev %d:%d, t_tid %u, inode %llu, i_sync_tid %u, error %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->t_tid, __entry->i_ino, __entry->i_sync_tid,
                  __entry->error)
        );

TRACE_EVENT(ext4_fc_track_range,
        TP_PROTO(handle_t *handle, struct inode *inode,
                 long start, long end, int ret),

        TP_ARGS(handle, inode, start, end, ret),

        TP_STRUCT__entry(
                __field(u64, i_ino)
                __field(long, start)
                __field(long, end)
                __field(dev_t, dev)
                __field(tid_t, t_tid)
                __field(tid_t, i_sync_tid)
                __field(int, error)
        ),

        TP_fast_assign(
                struct ext4_inode_info *ei = EXT4_I(inode);

                __entry->dev = inode->i_sb->s_dev;
                __entry->t_tid = handle->h_transaction->t_tid;
                __entry->i_ino = inode->i_ino;
                __entry->i_sync_tid = ei->i_sync_tid;
                __entry->start = start;
                __entry->end = end;
                __entry->error = ret;
        ),

        TP_printk("dev %d:%d, t_tid %u, inode %llu, i_sync_tid %u, error %d, start %ld, end %ld",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->t_tid, __entry->i_ino, __entry->i_sync_tid,
                  __entry->error, __entry->start, __entry->end)
        );

TRACE_EVENT(ext4_fc_cleanup,
        TP_PROTO(journal_t *journal, int full, tid_t tid),

        TP_ARGS(journal, full, tid),

        TP_STRUCT__entry(
                __field(dev_t, dev)
                __field(int, j_fc_off)
                __field(int, full)
                __field(tid_t, tid)
        ),

        TP_fast_assign(
                struct super_block *sb = journal->j_private;

                __entry->dev = sb->s_dev;
                __entry->j_fc_off = journal->j_fc_off;
                __entry->full = full;
                __entry->tid = tid;
        ),

        TP_printk("dev %d,%d, j_fc_off %d, full %d, tid %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->j_fc_off, __entry->full, __entry->tid)
        );

TRACE_EVENT(ext4_update_sb,
        TP_PROTO(struct super_block *sb, ext4_fsblk_t fsblk,
                 unsigned int flags),

        TP_ARGS(sb, fsblk, flags),

        TP_STRUCT__entry(
                __field(dev_t,                dev)
                __field(ext4_fsblk_t,        fsblk)
                __field(unsigned int,        flags)
        ),

        TP_fast_assign(
                __entry->dev        = sb->s_dev;
                __entry->fsblk        = fsblk;
                __entry->flags        = flags;
        ),

        TP_printk("dev %d,%d fsblk %llu flags %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->fsblk, __entry->flags)
);

TRACE_EVENT(ext4_move_extent_enter,
        TP_PROTO(struct inode *orig_inode, struct ext4_map_blocks *orig_map,
                 struct inode *donor_inode, ext4_lblk_t donor_lblk),

        TP_ARGS(orig_inode, orig_map, donor_inode, donor_lblk),

        TP_STRUCT__entry(
                __field(u64, orig_ino)
                __field(u64, donor_ino)
                __field(dev_t, dev)
                __field(ext4_lblk_t, orig_lblk)
                __field(unsigned int, orig_flags)
                __field(ext4_lblk_t, donor_lblk)
                __field(unsigned int, len)
        ),

        TP_fast_assign(
                __entry->dev                = orig_inode->i_sb->s_dev;
                __entry->orig_ino        = orig_inode->i_ino;
                __entry->orig_lblk        = orig_map->m_lblk;
                __entry->orig_flags        = orig_map->m_flags;
                __entry->donor_ino        = donor_inode->i_ino;
                __entry->donor_lblk        = donor_lblk;
                __entry->len                = orig_map->m_len;
        ),

        TP_printk("dev %d,%d origin ino %llu lblk %u flags %s donor ino %llu lblk %u len %u",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->orig_ino,  __entry->orig_lblk,
                  show_mflags(__entry->orig_flags),
                  __entry->donor_ino,  __entry->donor_lblk,
                  __entry->len)
);

TRACE_EVENT(ext4_move_extent_exit,
        TP_PROTO(struct inode *orig_inode, ext4_lblk_t orig_lblk,
                 struct inode *donor_inode, ext4_lblk_t donor_lblk,
                 unsigned int m_len, u64 move_len, int move_type, int ret),

        TP_ARGS(orig_inode, orig_lblk, donor_inode, donor_lblk, m_len,
                move_len, move_type, ret),

        TP_STRUCT__entry(
                __field(u64, orig_ino)
                __field(u64, donor_ino)
                __field(u64, move_len)
                __field(dev_t, dev)
                __field(ext4_lblk_t, orig_lblk)
                __field(ext4_lblk_t, donor_lblk)
                __field(unsigned int, m_len)
                __field(int, move_type)
                __field(int, ret)
        ),

        TP_fast_assign(
                __entry->dev                = orig_inode->i_sb->s_dev;
                __entry->orig_ino        = orig_inode->i_ino;
                __entry->orig_lblk        = orig_lblk;
                __entry->donor_ino        = donor_inode->i_ino;
                __entry->donor_lblk        = donor_lblk;
                __entry->m_len                = m_len;
                __entry->move_len        = move_len;
                __entry->move_type        = move_type;
                __entry->ret                = ret;
        ),

        TP_printk("dev %d,%d origin ino %llu lblk %u donor ino %llu lblk %u m_len %u, move_len %llu type %d ret %d",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  __entry->orig_ino,  __entry->orig_lblk,
                  __entry->donor_ino,  __entry->donor_lblk,
                  __entry->m_len, __entry->move_len, __entry->move_type,
                  __entry->ret)
);

#endif /* _TRACE_EXT4_H */

/* This part must be outside protection */
#include <trace/define_trace.h>


















































































 21034 





   17 






   17 































 21034 
 21034 

 21034 
 21034 












    6 

























   95 



   17 

   95 
   95 


    9 



   95 



















































    2 

















   29 























   29 






   28 






   28 



   27 


   26 










   29 

















 1502 


























    7 


    7 












    4 












    4 


    4 









    9 

    6 




   14 

   11 


















   54 












   30 


   30 



    9 


    7 

    9 

    7 

    5 















    3 
    1 














    1 

    1 





    4 






    4 
    3 

    2 





    2 






    1 

    3 


   30 
   54 















    6 


    6 




   13 


























   13 









   13 
   13 

   10 

    4 
    4 



    6 



    6 



    6 




    6 


    6 




































































    2 













    2 


    2 



    2 


    2 



















































    2 











    2 






    2 







    2 



    2 

    2 
    2 










    2 

























































































































    2 








    2 












































































    2 



















   55 


  308 







  308 


   55 


























    6 


  153 



    9 

    3 


  147 




    6 



































   87 

   11 


   43 


   21 
   21 








   21 

   86 
   53 
   12 
   89 















   92 





   89 
   89 









    3 
    2 
    1 
    1 


    1 
    1 
   81 








   92 
















  226 


  135 

  226 















   65 














  148 












   13 












    2 

    1 


    1 


    1 





















  315 




    1 

    1 


    2 





















    4 


    3 
















    3 




    1 






    1 



    2 


    2 



    1 


  315 


    3 

    2 


    2 



    2 

    1 



    9 
    2 


    1 


    1 



    7 


    6 
    1 
    5 



    4 
    3 
    2 

    1 


    2 



    1 

    1 
    2 




















 2354 

















 2411 
   65 



   64 

 2411 















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
// SPDX-License-Identifier: GPL-2.0-or-later
/* Common capabilities, needed by capability.o.
 */

#include <linux/capability.h>
#include <linux/audit.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/lsm_hooks.h>
#include <linux/file.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/swap.h>
#include <linux/skbuff.h>
#include <linux/netlink.h>
#include <linux/ptrace.h>
#include <linux/xattr.h>
#include <linux/hugetlb.h>
#include <linux/mount.h>
#include <linux/sched.h>
#include <linux/prctl.h>
#include <linux/securebits.h>
#include <linux/user_namespace.h>
#include <linux/binfmts.h>
#include <linux/personality.h>
#include <linux/mnt_idmapping.h>
#include <uapi/linux/lsm.h>

#define CREATE_TRACE_POINTS
#include <trace/events/capability.h>

/*
 * If a non-root user executes a setuid-root binary in
 * !secure(SECURE_NOROOT) mode, then we raise capabilities.
 * However if fE is also set, then the intent is for only
 * the file capabilities to be applied, and the setuid-root
 * bit is left on either to change the uid (plausible) or
 * to get full privilege on a kernel without file capabilities
 * support.  So in that case we do not raise capabilities.
 *
 * Warn if that happens, once per boot.
 */
static void warn_setuid_and_fcaps_mixed(const char *fname)
{
        static int warned;
        if (!warned) {
                printk(KERN_INFO "warning: `%s' has both setuid-root and"
                        " effective capabilities. Therefore not raising all"
                        " capabilities.\n", fname);
                warned = 1;
        }
}

/**
 * cap_capable_helper - Determine whether a task has a particular effective
 * capability.
 * @cred: The credentials to use
 * @target_ns:  The user namespace of the resource being accessed
 * @cred_ns:  The user namespace of the credentials
 * @cap: The capability to check for
 *
 * Determine whether the nominated task has the specified capability amongst
 * its effective set, returning 0 if it does, -ve if it does not.
 *
 * See cap_capable for more details.
 */
static inline int cap_capable_helper(const struct cred *cred,
                                     struct user_namespace *target_ns,
                                     const struct user_namespace *cred_ns,
                                     int cap)
{
        struct user_namespace *ns = target_ns;

        /* See if cred has the capability in the target user namespace
         * by examining the target user namespace and all of the target
         * user namespace's parents.
         */
        for (;;) {
                /* Do we have the necessary capabilities? */
                if (likely(ns == cred_ns))
                        return cap_raised(cred->cap_effective, cap) ? 0 : -EPERM;

                /*
                 * If we're already at a lower level than we're looking for,
                 * we're done searching.
                 */
                if (ns->level <= cred_ns->level)
                        return -EPERM;

                /* 
                 * The owner of the user namespace in the parent of the
                 * user namespace has all caps.
                 */
                if ((ns->parent == cred_ns) && uid_eq(ns->owner, cred->euid))
                        return 0;

                /*
                 * If you have a capability in a parent user ns, then you have
                 * it over all children user namespaces as well.
                 */
                ns = ns->parent;
        }

        /* We never get here */
}

/**
 * cap_capable - Determine whether a task has a particular effective capability
 * @cred: The credentials to use
 * @target_ns:  The user namespace of the resource being accessed
 * @cap: The capability to check for
 * @opts: Bitmask of options defined in include/linux/security.h (unused)
 *
 * Determine whether the nominated task has the specified capability amongst
 * its effective set, returning 0 if it does, -ve if it does not.
 *
 * NOTE WELL: cap_capable() has reverse semantics to the capable() call
 * and friends. That is cap_capable() returns an int 0 when a task has
 * a capability, while the kernel's capable(), has_ns_capability(),
 * has_ns_capability_noaudit(), and has_capability_noaudit() return a
 * bool true (1) for this case.
 */
int cap_capable(const struct cred *cred, struct user_namespace *target_ns,
                int cap, unsigned int opts)
{
        const struct user_namespace *cred_ns = cred->user_ns;
        int ret = cap_capable_helper(cred, target_ns, cred_ns, cap);

        trace_cap_capable(cred, target_ns, cred_ns, cap, ret);
        return ret;
}

/**
 * cap_settime - Determine whether the current process may set the system clock
 * @ts: The time to set
 * @tz: The timezone to set
 *
 * Determine whether the current process may set the system clock and timezone
 * information, returning 0 if permission granted, -ve if denied.
 */
int cap_settime(const struct timespec64 *ts, const struct timezone *tz)
{
        if (!capable(CAP_SYS_TIME))
                return -EPERM;
        return 0;
}

/**
 * cap_ptrace_access_check - Determine whether the current process may access
 *                           another
 * @child: The process to be accessed
 * @mode: The mode of attachment.
 *
 * If we are in the same or an ancestor user_ns and have all the target
 * task's capabilities, then ptrace access is allowed.
 * If we have the ptrace capability to the target user_ns, then ptrace
 * access is allowed.
 * Else denied.
 *
 * Determine whether a process may access another, returning 0 if permission
 * granted, -ve if denied.
 */
int cap_ptrace_access_check(struct task_struct *child, unsigned int mode)
{
        int ret = 0;
        const struct cred *cred, *child_cred;
        const kernel_cap_t *caller_caps;

        rcu_read_lock();
        cred = current_cred();
        child_cred = __task_cred(child);
        if (mode & PTRACE_MODE_FSCREDS)
                caller_caps = &cred->cap_effective;
        else
                caller_caps = &cred->cap_permitted;
        if (cred->user_ns == child_cred->user_ns &&
            cap_issubset(child_cred->cap_permitted, *caller_caps))
                goto out;
        if (ns_capable(child_cred->user_ns, CAP_SYS_PTRACE))
                goto out;
        ret = -EPERM;
out:
        rcu_read_unlock();
        return ret;
}

/**
 * cap_ptrace_traceme - Determine whether another process may trace the current
 * @parent: The task proposed to be the tracer
 *
 * If parent is in the same or an ancestor user_ns and has all current's
 * capabilities, then ptrace access is allowed.
 * If parent has the ptrace capability to current's user_ns, then ptrace
 * access is allowed.
 * Else denied.
 *
 * Determine whether the nominated task is permitted to trace the current
 * process, returning 0 if permission is granted, -ve if denied.
 */
int cap_ptrace_traceme(struct task_struct *parent)
{
        int ret = 0;
        const struct cred *cred, *child_cred;

        rcu_read_lock();
        cred = __task_cred(parent);
        child_cred = current_cred();
        if (cred->user_ns == child_cred->user_ns &&
            cap_issubset(child_cred->cap_permitted, cred->cap_permitted))
                goto out;
        if (has_ns_capability(parent, child_cred->user_ns, CAP_SYS_PTRACE))
                goto out;
        ret = -EPERM;
out:
        rcu_read_unlock();
        return ret;
}

/**
 * cap_capget - Retrieve a task's capability sets
 * @target: The task from which to retrieve the capability sets
 * @effective: The place to record the effective set
 * @inheritable: The place to record the inheritable set
 * @permitted: The place to record the permitted set
 *
 * This function retrieves the capabilities of the nominated task and returns
 * them to the caller.
 */
int cap_capget(const struct task_struct *target, kernel_cap_t *effective,
               kernel_cap_t *inheritable, kernel_cap_t *permitted)
{
        const struct cred *cred;

        /* Derived from kernel/capability.c:sys_capget. */
        rcu_read_lock();
        cred = __task_cred(target);
        *effective   = cred->cap_effective;
        *inheritable = cred->cap_inheritable;
        *permitted   = cred->cap_permitted;
        rcu_read_unlock();
        return 0;
}

/*
 * Determine whether the inheritable capabilities are limited to the old
 * permitted set.  Returns 1 if they are limited, 0 if they are not.
 */
static inline int cap_inh_is_capped(void)
{
        /* they are so limited unless the current task has the CAP_SETPCAP
         * capability
         */
        if (cap_capable(current_cred(), current_cred()->user_ns,
                        CAP_SETPCAP, CAP_OPT_NONE) == 0)
                return 0;
        return 1;
}

/**
 * cap_capset - Validate and apply proposed changes to current's capabilities
 * @new: The proposed new credentials; alterations should be made here
 * @old: The current task's current credentials
 * @effective: A pointer to the proposed new effective capabilities set
 * @inheritable: A pointer to the proposed new inheritable capabilities set
 * @permitted: A pointer to the proposed new permitted capabilities set
 *
 * This function validates and applies a proposed mass change to the current
 * process's capability sets.  The changes are made to the proposed new
 * credentials, and assuming no error, will be committed by the caller of LSM.
 */
int cap_capset(struct cred *new,
               const struct cred *old,
               const kernel_cap_t *effective,
               const kernel_cap_t *inheritable,
               const kernel_cap_t *permitted)
{
        if (cap_inh_is_capped() &&
            !cap_issubset(*inheritable,
                          cap_combine(old->cap_inheritable,
                                      old->cap_permitted)))
                /* incapable of using this inheritable set */
                return -EPERM;

        if (!cap_issubset(*inheritable,
                          cap_combine(old->cap_inheritable,
                                      old->cap_bset)))
                /* no new pI capabilities outside bounding set */
                return -EPERM;

        /* verify restrictions on target's new Permitted set */
        if (!cap_issubset(*permitted, old->cap_permitted))
                return -EPERM;

        /* verify the _new_Effective_ is a subset of the _new_Permitted_ */
        if (!cap_issubset(*effective, *permitted))
                return -EPERM;

        new->cap_effective   = *effective;
        new->cap_inheritable = *inheritable;
        new->cap_permitted   = *permitted;

        /*
         * Mask off ambient bits that are no longer both permitted and
         * inheritable.
         */
        new->cap_ambient = cap_intersect(new->cap_ambient,
                                         cap_intersect(*permitted,
                                                       *inheritable));
        if (WARN_ON(!cap_ambient_invariant_ok(new)))
                return -EINVAL;
        return 0;
}

/**
 * cap_inode_need_killpriv - Determine if inode change affects privileges
 * @dentry: The inode/dentry in being changed with change marked ATTR_KILL_PRIV
 *
 * Determine if an inode having a change applied that's marked ATTR_KILL_PRIV
 * affects the security markings on that inode, and if it is, should
 * inode_killpriv() be invoked or the change rejected.
 *
 * Return: 1 if security.capability has a value, meaning inode_killpriv()
 * is required, 0 otherwise, meaning inode_killpriv() is not required.
 */
int cap_inode_need_killpriv(struct dentry *dentry)
{
        struct inode *inode = d_backing_inode(dentry);
        int error;

        error = __vfs_getxattr(dentry, inode, XATTR_NAME_CAPS, NULL, 0);
        return error > 0;
}

/**
 * cap_inode_killpriv - Erase the security markings on an inode
 *
 * @idmap:        idmap of the mount the inode was found from
 * @dentry:        The inode/dentry to alter
 *
 * Erase the privilege-enhancing security markings on an inode.
 *
 * If the inode has been found through an idmapped mount the idmap of
 * the vfsmount must be passed through @idmap. This function will then
 * take care to map the inode according to @idmap before checking
 * permissions. On non-idmapped mounts or if permission checking is to be
 * performed on the raw inode simply pass @nop_mnt_idmap.
 *
 * Return: 0 if successful, -ve on error.
 */
int cap_inode_killpriv(struct mnt_idmap *idmap, struct dentry *dentry)
{
        int error;

        error = __vfs_removexattr(idmap, dentry, XATTR_NAME_CAPS);
        if (error == -EOPNOTSUPP)
                error = 0;
        return error;
}

/**
 * kuid_root_in_ns - check whether the given kuid is root in the given ns
 * @kuid: the kuid to be tested
 * @ns: the user namespace to test against
 *
 * Returns true if @kuid represents the root user in @ns, false otherwise.
 */
static bool kuid_root_in_ns(kuid_t kuid, struct user_namespace *ns)
{
        for (;; ns = ns->parent) {
                if (from_kuid(ns, kuid) == 0)
                        return true;
                if (ns == &init_user_ns)
                        break;
        }

        return false;
}

static bool vfsuid_root_in_currentns(vfsuid_t vfsuid)
{
        kuid_t kuid;

        if (!vfsuid_valid(vfsuid))
                return false;
        kuid = vfsuid_into_kuid(vfsuid);
        return kuid_root_in_ns(kuid, current_user_ns());
}

static __u32 sansflags(__u32 m)
{
        return m & ~VFS_CAP_FLAGS_EFFECTIVE;
}

static bool is_v2header(int size, const struct vfs_cap_data *cap)
{
        if (size != XATTR_CAPS_SZ_2)
                return false;
        return sansflags(le32_to_cpu(cap->magic_etc)) == VFS_CAP_REVISION_2;
}

static bool is_v3header(int size, const struct vfs_cap_data *cap)
{
        if (size != XATTR_CAPS_SZ_3)
                return false;
        return sansflags(le32_to_cpu(cap->magic_etc)) == VFS_CAP_REVISION_3;
}

/*
 * getsecurity: We are called for security.* before any attempt to read the
 * xattr from the inode itself.
 *
 * This gives us a chance to read the on-disk value and convert it.  If we
 * return -EOPNOTSUPP, then vfs_getxattr() will call the i_op handler.
 *
 * Note we are not called by vfs_getxattr_alloc(), but that is only called
 * by the integrity subsystem, which really wants the unconverted values -
 * so that's good.
 */
int cap_inode_getsecurity(struct mnt_idmap *idmap,
                          struct inode *inode, const char *name, void **buffer,
                          bool alloc)
{
        int size;
        kuid_t kroot;
        vfsuid_t vfsroot;
        u32 nsmagic, magic;
        uid_t root, mappedroot;
        char *tmpbuf = NULL;
        struct vfs_cap_data *cap;
        struct vfs_ns_cap_data *nscap = NULL;
        struct dentry *dentry;
        struct user_namespace *fs_ns;

        if (strcmp(name, "capability") != 0)
                return -EOPNOTSUPP;

        dentry = d_find_any_alias(inode);
        if (!dentry)
                return -EINVAL;
        size = vfs_getxattr_alloc(idmap, dentry, XATTR_NAME_CAPS, &tmpbuf,
                                  sizeof(struct vfs_ns_cap_data), GFP_NOFS);
        dput(dentry);
        /* gcc11 complains if we don't check for !tmpbuf */
        if (size < 0 || !tmpbuf)
                goto out_free;

        fs_ns = inode->i_sb->s_user_ns;
        cap = (struct vfs_cap_data *) tmpbuf;
        if (is_v2header(size, cap)) {
                root = 0;
        } else if (is_v3header(size, cap)) {
                nscap = (struct vfs_ns_cap_data *) tmpbuf;
                root = le32_to_cpu(nscap->rootid);
        } else {
                size = -EINVAL;
                goto out_free;
        }

        kroot = make_kuid(fs_ns, root);

        /* If this is an idmapped mount shift the kuid. */
        vfsroot = make_vfsuid(idmap, fs_ns, kroot);

        /* If the root kuid maps to a valid uid in current ns, then return
         * this as a nscap. */
        mappedroot = from_kuid(current_user_ns(), vfsuid_into_kuid(vfsroot));
        if (mappedroot != (uid_t)-1 && mappedroot != (uid_t)0) {
                size = sizeof(struct vfs_ns_cap_data);
                if (alloc) {
                        if (!nscap) {
                                /* v2 -> v3 conversion */
                                nscap = kzalloc(size, GFP_ATOMIC);
                                if (!nscap) {
                                        size = -ENOMEM;
                                        goto out_free;
                                }
                                nsmagic = VFS_CAP_REVISION_3;
                                magic = le32_to_cpu(cap->magic_etc);
                                if (magic & VFS_CAP_FLAGS_EFFECTIVE)
                                        nsmagic |= VFS_CAP_FLAGS_EFFECTIVE;
                                memcpy(&nscap->data, &cap->data, sizeof(__le32) * 2 * VFS_CAP_U32);
                                nscap->magic_etc = cpu_to_le32(nsmagic);
                        } else {
                                /* use allocated v3 buffer */
                                tmpbuf = NULL;
                        }
                        nscap->rootid = cpu_to_le32(mappedroot);
                        *buffer = nscap;
                }
                goto out_free;
        }

        if (!vfsuid_root_in_currentns(vfsroot)) {
                size = -EOVERFLOW;
                goto out_free;
        }

        /* This comes from a parent namespace.  Return as a v2 capability */
        size = sizeof(struct vfs_cap_data);
        if (alloc) {
                if (nscap) {
                        /* v3 -> v2 conversion */
                        cap = kzalloc(size, GFP_ATOMIC);
                        if (!cap) {
                                size = -ENOMEM;
                                goto out_free;
                        }
                        magic = VFS_CAP_REVISION_2;
                        nsmagic = le32_to_cpu(nscap->magic_etc);
                        if (nsmagic & VFS_CAP_FLAGS_EFFECTIVE)
                                magic |= VFS_CAP_FLAGS_EFFECTIVE;
                        memcpy(&cap->data, &nscap->data, sizeof(__le32) * 2 * VFS_CAP_U32);
                        cap->magic_etc = cpu_to_le32(magic);
                } else {
                        /* use unconverted v2 */
                        tmpbuf = NULL;
                }
                *buffer = cap;
        }
out_free:
        kfree(tmpbuf);
        return size;
}

/**
 * rootid_from_xattr - translate root uid of vfs caps
 *
 * @value:        vfs caps value which may be modified by this function
 * @size:        size of @ivalue
 * @task_ns:        user namespace of the caller
 */
static vfsuid_t rootid_from_xattr(const void *value, size_t size,
                                  struct user_namespace *task_ns)
{
        const struct vfs_ns_cap_data *nscap = value;
        uid_t rootid = 0;

        if (size == XATTR_CAPS_SZ_3)
                rootid = le32_to_cpu(nscap->rootid);

        return VFSUIDT_INIT(make_kuid(task_ns, rootid));
}

static bool validheader(size_t size, const struct vfs_cap_data *cap)
{
        return is_v2header(size, cap) || is_v3header(size, cap);
}

/**
 * cap_convert_nscap - check vfs caps
 *
 * @idmap:        idmap of the mount the inode was found from
 * @dentry:        used to retrieve inode to check permissions on
 * @ivalue:        vfs caps value which may be modified by this function
 * @size:        size of @ivalue
 *
 * User requested a write of security.capability.  If needed, update the
 * xattr to change from v2 to v3, or to fixup the v3 rootid.
 *
 * If the inode has been found through an idmapped mount the idmap of
 * the vfsmount must be passed through @idmap. This function will then
 * take care to map the inode according to @idmap before checking
 * permissions. On non-idmapped mounts or if permission checking is to be
 * performed on the raw inode simply pass @nop_mnt_idmap.
 *
 * Return: On success, return the new size; on error, return < 0.
 */
int cap_convert_nscap(struct mnt_idmap *idmap, struct dentry *dentry,
                      const void **ivalue, size_t size)
{
        struct vfs_ns_cap_data *nscap;
        uid_t nsrootid;
        const struct vfs_cap_data *cap = *ivalue;
        __u32 magic, nsmagic;
        struct inode *inode = d_backing_inode(dentry);
        struct user_namespace *task_ns = current_user_ns(),
                *fs_ns = inode->i_sb->s_user_ns;
        kuid_t rootid;
        vfsuid_t vfsrootid;
        size_t newsize;

        if (!*ivalue)
                return -EINVAL;
        if (!validheader(size, cap))
                return -EINVAL;
        if (!capable_wrt_inode_uidgid(idmap, inode, CAP_SETFCAP))
                return -EPERM;
        if (size == XATTR_CAPS_SZ_2 && (idmap == &nop_mnt_idmap))
                if (ns_capable(inode->i_sb->s_user_ns, CAP_SETFCAP))
                        /* user is privileged, just write the v2 */
                        return size;

        vfsrootid = rootid_from_xattr(*ivalue, size, task_ns);
        if (!vfsuid_valid(vfsrootid))
                return -EINVAL;

        rootid = from_vfsuid(idmap, fs_ns, vfsrootid);
        if (!uid_valid(rootid))
                return -EINVAL;

        nsrootid = from_kuid(fs_ns, rootid);
        if (nsrootid == -1)
                return -EINVAL;

        newsize = sizeof(struct vfs_ns_cap_data);
        nscap = kmalloc(newsize, GFP_ATOMIC);
        if (!nscap)
                return -ENOMEM;
        nscap->rootid = cpu_to_le32(nsrootid);
        nsmagic = VFS_CAP_REVISION_3;
        magic = le32_to_cpu(cap->magic_etc);
        if (magic & VFS_CAP_FLAGS_EFFECTIVE)
                nsmagic |= VFS_CAP_FLAGS_EFFECTIVE;
        nscap->magic_etc = cpu_to_le32(nsmagic);
        memcpy(&nscap->data, &cap->data, sizeof(__le32) * 2 * VFS_CAP_U32);

        *ivalue = nscap;
        return newsize;
}

/*
 * Calculate the new process capability sets from the capability sets attached
 * to a file.
 */
static inline int bprm_caps_from_vfs_caps(struct cpu_vfs_cap_data *caps,
                                          struct linux_binprm *bprm,
                                          bool *effective,
                                          bool *has_fcap)
{
        struct cred *new = bprm->cred;
        int ret = 0;

        if (caps->magic_etc & VFS_CAP_FLAGS_EFFECTIVE)
                *effective = true;

        if (caps->magic_etc & VFS_CAP_REVISION_MASK)
                *has_fcap = true;

        /*
         * pP' = (X & fP) | (pI & fI)
         * The addition of pA' is handled later.
         */
        new->cap_permitted.val =
                (new->cap_bset.val & caps->permitted.val) |
                (new->cap_inheritable.val & caps->inheritable.val);

        if (caps->permitted.val & ~new->cap_permitted.val)
                /* insufficient to execute correctly */
                ret = -EPERM;

        /*
         * For legacy apps, with no internal support for recognizing they
         * do not have enough capabilities, we return an error if they are
         * missing some "forced" (aka file-permitted) capabilities.
         */
        return *effective ? ret : 0;
}

/**
 * get_vfs_caps_from_disk - retrieve vfs caps from disk
 *
 * @idmap:        idmap of the mount the inode was found from
 * @dentry:        dentry from which @inode is retrieved
 * @cpu_caps:        vfs capabilities
 *
 * Extract the on-exec-apply capability sets for an executable file.
 *
 * If the inode has been found through an idmapped mount the idmap of
 * the vfsmount must be passed through @idmap. This function will then
 * take care to map the inode according to @idmap before checking
 * permissions. On non-idmapped mounts or if permission checking is to be
 * performed on the raw inode simply pass @nop_mnt_idmap.
 */
int get_vfs_caps_from_disk(struct mnt_idmap *idmap,
                           const struct dentry *dentry,
                           struct cpu_vfs_cap_data *cpu_caps)
{
        struct inode *inode = d_backing_inode(dentry);
        __u32 magic_etc;
        int size;
        struct vfs_ns_cap_data data, *nscaps = &data;
        struct vfs_cap_data *caps = (struct vfs_cap_data *) &data;
        kuid_t rootkuid;
        vfsuid_t rootvfsuid;
        struct user_namespace *fs_ns;

        memset(cpu_caps, 0, sizeof(struct cpu_vfs_cap_data));

        if (!inode)
                return -ENODATA;

        fs_ns = inode->i_sb->s_user_ns;
        size = __vfs_getxattr((struct dentry *)dentry, inode,
                              XATTR_NAME_CAPS, &data, XATTR_CAPS_SZ);
        if (size == -ENODATA || size == -EOPNOTSUPP)
                /* no data, that's ok */
                return -ENODATA;

        if (size < 0)
                return size;

        if (size < sizeof(magic_etc))
                return -EINVAL;

        cpu_caps->magic_etc = magic_etc = le32_to_cpu(caps->magic_etc);

        rootkuid = make_kuid(fs_ns, 0);
        switch (magic_etc & VFS_CAP_REVISION_MASK) {
        case VFS_CAP_REVISION_1:
                if (size != XATTR_CAPS_SZ_1)
                        return -EINVAL;
                break;
        case VFS_CAP_REVISION_2:
                if (size != XATTR_CAPS_SZ_2)
                        return -EINVAL;
                break;
        case VFS_CAP_REVISION_3:
                if (size != XATTR_CAPS_SZ_3)
                        return -EINVAL;
                rootkuid = make_kuid(fs_ns, le32_to_cpu(nscaps->rootid));
                break;

        default:
                return -EINVAL;
        }

        rootvfsuid = make_vfsuid(idmap, fs_ns, rootkuid);
        if (!vfsuid_valid(rootvfsuid))
                return -ENODATA;

        /* Limit the caps to the mounter of the filesystem
         * or the more limited uid specified in the xattr.
         */
        if (!vfsuid_root_in_currentns(rootvfsuid))
                return -ENODATA;

        cpu_caps->permitted.val = le32_to_cpu(caps->data[0].permitted);
        cpu_caps->inheritable.val = le32_to_cpu(caps->data[0].inheritable);

        /*
         * Rev1 had just a single 32-bit word, later expanded
         * to a second one for the high bits
         */
        if ((magic_etc & VFS_CAP_REVISION_MASK) != VFS_CAP_REVISION_1) {
                cpu_caps->permitted.val += (u64)le32_to_cpu(caps->data[1].permitted) << 32;
                cpu_caps->inheritable.val += (u64)le32_to_cpu(caps->data[1].inheritable) << 32;
        }

        cpu_caps->permitted.val &= CAP_VALID_MASK;
        cpu_caps->inheritable.val &= CAP_VALID_MASK;

        cpu_caps->rootid = vfsuid_into_kuid(rootvfsuid);

        return 0;
}

/*
 * Attempt to get the on-exec apply capability sets for an executable file from
 * its xattrs and, if present, apply them to the proposed credentials being
 * constructed by execve().
 */
static int get_file_caps(struct linux_binprm *bprm, const struct file *file,
                         bool *effective, bool *has_fcap)
{
        int rc = 0;
        struct cpu_vfs_cap_data vcaps;

        cap_clear(bprm->cred->cap_permitted);

        if (!file_caps_enabled)
                return 0;

        if (!mnt_may_suid(file->f_path.mnt))
                return 0;

        /*
         * This check is redundant with mnt_may_suid() but is kept to make
         * explicit that capability bits are limited to s_user_ns and its
         * descendants.
         */
        if (!current_in_userns(file->f_path.mnt->mnt_sb->s_user_ns))
                return 0;

        rc = get_vfs_caps_from_disk(file_mnt_idmap(file),
                                    file->f_path.dentry, &vcaps);
        if (rc < 0) {
                if (rc == -EINVAL)
                        printk(KERN_NOTICE "Invalid argument reading file caps for %s\n",
                                        bprm->filename);
                else if (rc == -ENODATA)
                        rc = 0;
                goto out;
        }

        rc = bprm_caps_from_vfs_caps(&vcaps, bprm, effective, has_fcap);

out:
        if (rc)
                cap_clear(bprm->cred->cap_permitted);

        return rc;
}

static inline bool root_privileged(void) { return !issecure(SECURE_NOROOT); }

static inline bool __is_real(kuid_t uid, struct cred *cred)
{ return uid_eq(cred->uid, uid); }

static inline bool __is_eff(kuid_t uid, struct cred *cred)
{ return uid_eq(cred->euid, uid); }

static inline bool __is_suid(kuid_t uid, struct cred *cred)
{ return !__is_real(uid, cred) && __is_eff(uid, cred); }

/*
 * handle_privileged_root - Handle case of privileged root
 * @bprm: The execution parameters, including the proposed creds
 * @has_fcap: Are any file capabilities set?
 * @effective: Do we have effective root privilege?
 * @root_uid: This namespace' root UID WRT initial USER namespace
 *
 * Handle the case where root is privileged and hasn't been neutered by
 * SECURE_NOROOT.  If file capabilities are set, they won't be combined with
 * set UID root and nothing is changed.  If we are root, cap_permitted is
 * updated.  If we have become set UID root, the effective bit is set.
 */
static void handle_privileged_root(struct linux_binprm *bprm, bool has_fcap,
                                   bool *effective, kuid_t root_uid)
{
        const struct cred *old = current_cred();
        struct cred *new = bprm->cred;

        if (!root_privileged())
                return;
        /*
         * If the legacy file capability is set, then don't set privs
         * for a setuid root binary run by a non-root user.  Do set it
         * for a root user just to cause least surprise to an admin.
         */
        if (has_fcap && __is_suid(root_uid, new)) {
                warn_setuid_and_fcaps_mixed(bprm->filename);
                return;
        }
        /*
         * To support inheritance of root-permissions and suid-root
         * executables under compatibility mode, we override the
         * capability sets for the file.
         */
        if (__is_eff(root_uid, new) || __is_real(root_uid, new)) {
                /* pP' = (cap_bset & ~0) | (pI & ~0) */
                new->cap_permitted = cap_combine(old->cap_bset,
                                                 old->cap_inheritable);
        }
        /*
         * If only the real uid is 0, we do not set the effective bit.
         */
        if (__is_eff(root_uid, new))
                *effective = true;
}

#define __cap_gained(field, target, source) \
        !cap_issubset(target->cap_##field, source->cap_##field)
#define __cap_grew(target, source, cred) \
        !cap_issubset(cred->cap_##target, cred->cap_##source)
#define __cap_full(field, cred) \
        cap_issubset(CAP_FULL_SET, cred->cap_##field)

/*
 * 1) Audit candidate if current->cap_effective is set
 *
 * We do not bother to audit if 3 things are true:
 *   1) cap_effective has all caps
 *   2) we became root *OR* are were already root
 *   3) root is supposed to have all caps (SECURE_NOROOT)
 * Since this is just a normal root execing a process.
 *
 * Number 1 above might fail if you don't have a full bset, but I think
 * that is interesting information to audit.
 *
 * A number of other conditions require logging:
 * 2) something prevented setuid root getting all caps
 * 3) non-setuid root gets fcaps
 * 4) non-setuid root gets ambient
 */
static inline bool nonroot_raised_pE(struct cred *new, const struct cred *old,
                                     kuid_t root, bool has_fcap)
{
        bool ret = false;

        if ((__cap_grew(effective, ambient, new) &&
             !(__cap_full(effective, new) &&
               (__is_eff(root, new) || __is_real(root, new)) &&
               root_privileged())) ||
            (root_privileged() &&
             __is_suid(root, new) &&
             !__cap_full(effective, new)) ||
            (uid_eq(new->euid, old->euid) &&
             ((has_fcap &&
               __cap_gained(permitted, new, old)) ||
              __cap_gained(ambient, new, old))))

                ret = true;

        return ret;
}

/**
 * cap_bprm_creds_from_file - Set up the proposed credentials for execve().
 * @bprm: The execution parameters, including the proposed creds
 * @file: The file to pull the credentials from
 *
 * Set up the proposed credentials for a new execution context being
 * constructed by execve().  The proposed creds in @bprm->cred is altered,
 * which won't take effect immediately.
 *
 * Return: 0 if successful, -ve on error.
 */
int cap_bprm_creds_from_file(struct linux_binprm *bprm, const struct file *file)
{
        /* Process setpcap binaries and capabilities for uid 0 */
        const struct cred *old = current_cred();
        struct cred *new = bprm->cred;
        bool effective = false, has_fcap = false, id_changed;
        int ret;
        kuid_t root_uid;

        if (WARN_ON(!cap_ambient_invariant_ok(old)))
                return -EPERM;

        ret = get_file_caps(bprm, file, &effective, &has_fcap);
        if (ret < 0)
                return ret;

        root_uid = make_kuid(new->user_ns, 0);

        handle_privileged_root(bprm, has_fcap, &effective, root_uid);

        /* if we have fs caps, clear dangerous personality flags */
        if (__cap_gained(permitted, new, old))
                bprm->per_clear |= PER_CLEAR_ON_SETID;

        /* Don't let someone trace a set[ug]id/setpcap binary with the revised
         * credentials unless they have the appropriate permit.
         *
         * In addition, if NO_NEW_PRIVS, then ensure we get no new privs.
         */
        id_changed = !uid_eq(new->euid, old->euid) || !in_group_p(new->egid);

        if ((id_changed || __cap_gained(permitted, new, old)) &&
            ((bprm->unsafe & ~LSM_UNSAFE_PTRACE) ||
             !ptracer_capable(current, new->user_ns))) {
                /* downgrade; they get no more than they had, and maybe less */
                if (!ns_capable(new->user_ns, CAP_SETUID) ||
                    (bprm->unsafe & LSM_UNSAFE_NO_NEW_PRIVS)) {
                        new->euid = new->uid;
                        new->egid = new->gid;
                }
                new->cap_permitted = cap_intersect(new->cap_permitted,
                                                   old->cap_permitted);
        }

        new->suid = new->fsuid = new->euid;
        new->sgid = new->fsgid = new->egid;

        /* File caps or setid cancels ambient. */
        if (has_fcap || id_changed)
                cap_clear(new->cap_ambient);

        /*
         * Now that we've computed pA', update pP' to give:
         *   pP' = (X & fP) | (pI & fI) | pA'
         */
        new->cap_permitted = cap_combine(new->cap_permitted, new->cap_ambient);

        /*
         * Set pE' = (fE ? pP' : pA').  Because pA' is zero if fE is set,
         * this is the same as pE' = (fE ? pP' : 0) | pA'.
         */
        if (effective)
                new->cap_effective = new->cap_permitted;
        else
                new->cap_effective = new->cap_ambient;

        if (WARN_ON(!cap_ambient_invariant_ok(new)))
                return -EPERM;

        if (nonroot_raised_pE(new, old, root_uid, has_fcap)) {
                ret = audit_log_bprm_fcaps(bprm, new, old);
                if (ret < 0)
                        return ret;
        }

        new->securebits &= ~issecure_mask(SECURE_KEEP_CAPS);

        if (WARN_ON(!cap_ambient_invariant_ok(new)))
                return -EPERM;

        /* Check for privilege-elevated exec. */
        if (id_changed ||
            !uid_eq(new->euid, old->uid) ||
            !gid_eq(new->egid, old->gid) ||
            (!__is_real(root_uid, new) &&
             (effective ||
              __cap_grew(permitted, ambient, new))))
                bprm->secureexec = 1;

        return 0;
}

/**
 * cap_inode_setxattr - Determine whether an xattr may be altered
 * @dentry: The inode/dentry being altered
 * @name: The name of the xattr to be changed
 * @value: The value that the xattr will be changed to
 * @size: The size of value
 * @flags: The replacement flag
 *
 * Determine whether an xattr may be altered or set on an inode, returning 0 if
 * permission is granted, -ve if denied.
 *
 * This is used to make sure security xattrs don't get updated or set by those
 * who aren't privileged to do so.
 */
int cap_inode_setxattr(struct dentry *dentry, const char *name,
                       const void *value, size_t size, int flags)
{
        struct user_namespace *user_ns = dentry->d_sb->s_user_ns;

        /* Ignore non-security xattrs */
        if (strncmp(name, XATTR_SECURITY_PREFIX,
                        XATTR_SECURITY_PREFIX_LEN) != 0)
                return 0;

        /*
         * For XATTR_NAME_CAPS the check will be done in
         * cap_convert_nscap(), called by setxattr()
         */
        if (strcmp(name, XATTR_NAME_CAPS) == 0)
                return 0;

        if (!ns_capable(user_ns, CAP_SYS_ADMIN))
                return -EPERM;
        return 0;
}

/**
 * cap_inode_removexattr - Determine whether an xattr may be removed
 *
 * @idmap:        idmap of the mount the inode was found from
 * @dentry:        The inode/dentry being altered
 * @name:        The name of the xattr to be changed
 *
 * Determine whether an xattr may be removed from an inode, returning 0 if
 * permission is granted, -ve if denied.
 *
 * If the inode has been found through an idmapped mount the idmap of
 * the vfsmount must be passed through @idmap. This function will then
 * take care to map the inode according to @idmap before checking
 * permissions. On non-idmapped mounts or if permission checking is to be
 * performed on the raw inode simply pass @nop_mnt_idmap.
 *
 * This is used to make sure security xattrs don't get removed by those who
 * aren't privileged to remove them.
 */
int cap_inode_removexattr(struct mnt_idmap *idmap,
                          struct dentry *dentry, const char *name)
{
        struct user_namespace *user_ns = dentry->d_sb->s_user_ns;

        /* Ignore non-security xattrs */
        if (strncmp(name, XATTR_SECURITY_PREFIX,
                        XATTR_SECURITY_PREFIX_LEN) != 0)
                return 0;

        if (strcmp(name, XATTR_NAME_CAPS) == 0) {
                /* security.capability gets namespaced */
                struct inode *inode = d_backing_inode(dentry);
                if (!inode)
                        return -EINVAL;
                if (!capable_wrt_inode_uidgid(idmap, inode, CAP_SETFCAP))
                        return -EPERM;
                return 0;
        }

        if (!ns_capable(user_ns, CAP_SYS_ADMIN))
                return -EPERM;
        return 0;
}

/*
 * cap_emulate_setxuid() fixes the effective / permitted capabilities of
 * a process after a call to setuid, setreuid, or setresuid.
 *
 *  1) When set*uiding _from_ one of {r,e,s}uid == 0 _to_ all of
 *  {r,e,s}uid != 0, the permitted and effective capabilities are
 *  cleared.
 *
 *  2) When set*uiding _from_ euid == 0 _to_ euid != 0, the effective
 *  capabilities of the process are cleared.
 *
 *  3) When set*uiding _from_ euid != 0 _to_ euid == 0, the effective
 *  capabilities are set to the permitted capabilities.
 *
 *  fsuid is handled elsewhere. fsuid == 0 and {r,e,s}uid!= 0 should
 *  never happen.
 *
 *  -astor
 *
 * cevans - New behaviour, Oct '99
 * A process may, via prctl(), elect to keep its capabilities when it
 * calls setuid() and switches away from uid==0. Both permitted and
 * effective sets will be retained.
 * Without this change, it was impossible for a daemon to drop only some
 * of its privilege. The call to setuid(!=0) would drop all privileges!
 * Keeping uid 0 is not an option because uid 0 owns too many vital
 * files..
 * Thanks to Olaf Kirch and Peter Benie for spotting this.
 */
static inline void cap_emulate_setxuid(struct cred *new, const struct cred *old)
{
        kuid_t root_uid = make_kuid(old->user_ns, 0);

        if ((uid_eq(old->uid, root_uid) ||
             uid_eq(old->euid, root_uid) ||
             uid_eq(old->suid, root_uid)) &&
            (!uid_eq(new->uid, root_uid) &&
             !uid_eq(new->euid, root_uid) &&
             !uid_eq(new->suid, root_uid))) {
                if (!issecure(SECURE_KEEP_CAPS)) {
                        cap_clear(new->cap_permitted);
                        cap_clear(new->cap_effective);
                }

                /*
                 * Pre-ambient programs expect setresuid to nonroot followed
                 * by exec to drop capabilities.  We should make sure that
                 * this remains the case.
                 */
                cap_clear(new->cap_ambient);
        }
        if (uid_eq(old->euid, root_uid) && !uid_eq(new->euid, root_uid))
                cap_clear(new->cap_effective);
        if (!uid_eq(old->euid, root_uid) && uid_eq(new->euid, root_uid))
                new->cap_effective = new->cap_permitted;
}

/**
 * cap_task_fix_setuid - Fix up the results of setuid() call
 * @new: The proposed credentials
 * @old: The current task's current credentials
 * @flags: Indications of what has changed
 *
 * Fix up the results of setuid() call before the credential changes are
 * actually applied.
 *
 * Return: 0 to grant the changes, -ve to deny them.
 */
int cap_task_fix_setuid(struct cred *new, const struct cred *old, int flags)
{
        switch (flags) {
        case LSM_SETID_RE:
        case LSM_SETID_ID:
        case LSM_SETID_RES:
                /* juggle the capabilities to follow [RES]UID changes unless
                 * otherwise suppressed */
                if (!issecure(SECURE_NO_SETUID_FIXUP))
                        cap_emulate_setxuid(new, old);
                break;

        case LSM_SETID_FS:
                /* juggle the capabilities to follow FSUID changes, unless
                 * otherwise suppressed
                 *
                 * FIXME - is fsuser used for all CAP_FS_MASK capabilities?
                 *          if not, we might be a bit too harsh here.
                 */
                if (!issecure(SECURE_NO_SETUID_FIXUP)) {
                        kuid_t root_uid = make_kuid(old->user_ns, 0);
                        if (uid_eq(old->fsuid, root_uid) && !uid_eq(new->fsuid, root_uid))
                                new->cap_effective =
                                        cap_drop_fs_set(new->cap_effective);

                        if (!uid_eq(old->fsuid, root_uid) && uid_eq(new->fsuid, root_uid))
                                new->cap_effective =
                                        cap_raise_fs_set(new->cap_effective,
                                                         new->cap_permitted);
                }
                break;

        default:
                return -EINVAL;
        }

        return 0;
}

/*
 * Rationale: code calling task_setscheduler, task_setioprio, and
 * task_setnice, assumes that
 *   . if capable(cap_sys_nice), then those actions should be allowed
 *   . if not capable(cap_sys_nice), but acting on your own processes,
 *           then those actions should be allowed
 * This is insufficient now since you can call code without suid, but
 * yet with increased caps.
 * So we check for increased caps on the target process.
 */
static int cap_safe_nice(struct task_struct *p)
{
        int is_subset, ret = 0;

        rcu_read_lock();
        is_subset = cap_issubset(__task_cred(p)->cap_permitted,
                                 current_cred()->cap_permitted);
        if (!is_subset && !ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE))
                ret = -EPERM;
        rcu_read_unlock();

        return ret;
}

/**
 * cap_task_setscheduler - Determine if scheduler policy change is permitted
 * @p: The task to affect
 *
 * Determine if the requested scheduler policy change is permitted for the
 * specified task.
 *
 * Return: 0 if permission is granted, -ve if denied.
 */
int cap_task_setscheduler(struct task_struct *p)
{
        return cap_safe_nice(p);
}

/**
 * cap_task_setioprio - Determine if I/O priority change is permitted
 * @p: The task to affect
 * @ioprio: The I/O priority to set
 *
 * Determine if the requested I/O priority change is permitted for the specified
 * task.
 *
 * Return: 0 if permission is granted, -ve if denied.
 */
int cap_task_setioprio(struct task_struct *p, int ioprio)
{
        return cap_safe_nice(p);
}

/**
 * cap_task_setnice - Determine if task priority change is permitted
 * @p: The task to affect
 * @nice: The nice value to set
 *
 * Determine if the requested task priority change is permitted for the
 * specified task.
 *
 * Return: 0 if permission is granted, -ve if denied.
 */
int cap_task_setnice(struct task_struct *p, int nice)
{
        return cap_safe_nice(p);
}

/*
 * Implement PR_CAPBSET_DROP.  Attempt to remove the specified capability from
 * the current task's bounding set.  Returns 0 on success, -ve on error.
 */
static int cap_prctl_drop(unsigned long cap)
{
        struct cred *new;

        if (!ns_capable(current_user_ns(), CAP_SETPCAP))
                return -EPERM;
        if (!cap_valid(cap))
                return -EINVAL;

        new = prepare_creds();
        if (!new)
                return -ENOMEM;
        cap_lower(new->cap_bset, cap);
        return commit_creds(new);
}

/**
 * cap_task_prctl - Implement process control functions for this security module
 * @option: The process control function requested
 * @arg2: The argument data for this function
 * @arg3: The argument data for this function
 * @arg4: The argument data for this function
 * @arg5: The argument data for this function
 *
 * Allow process control functions (sys_prctl()) to alter capabilities; may
 * also deny access to other functions not otherwise implemented here.
 *
 * Return: 0 or +ve on success, -ENOSYS if this function is not implemented
 * here, other -ve on error.  If -ENOSYS is returned, sys_prctl() and other LSM
 * modules will consider performing the function.
 */
int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3,
                   unsigned long arg4, unsigned long arg5)
{
        const struct cred *old = current_cred();
        struct cred *new;

        switch (option) {
        case PR_CAPBSET_READ:
                if (!cap_valid(arg2))
                        return -EINVAL;
                return !!cap_raised(old->cap_bset, arg2);

        case PR_CAPBSET_DROP:
                return cap_prctl_drop(arg2);

        /*
         * The next four prctl's remain to assist with transitioning a
         * system from legacy UID=0 based privilege (when filesystem
         * capabilities are not in use) to a system using filesystem
         * capabilities only - as the POSIX.1e draft intended.
         *
         * Note:
         *
         *  PR_SET_SECUREBITS =
         *      issecure_mask(SECURE_KEEP_CAPS_LOCKED)
         *    | issecure_mask(SECURE_NOROOT)
         *    | issecure_mask(SECURE_NOROOT_LOCKED)
         *    | issecure_mask(SECURE_NO_SETUID_FIXUP)
         *    | issecure_mask(SECURE_NO_SETUID_FIXUP_LOCKED)
         *
         * will ensure that the current process and all of its
         * children will be locked into a pure
         * capability-based-privilege environment.
         */
        case PR_SET_SECUREBITS:
                if ((((old->securebits & SECURE_ALL_LOCKS) >> 1)
                     & (old->securebits ^ arg2))                        /*[1]*/
                    || ((old->securebits & SECURE_ALL_LOCKS & ~arg2))        /*[2]*/
                    || (arg2 & ~(SECURE_ALL_LOCKS | SECURE_ALL_BITS))        /*[3]*/
                        /*
                         * [1] no changing of bits that are locked
                         * [2] no unlocking of locks
                         * [3] no setting of unsupported bits
                         */
                    )
                        /* cannot change a locked bit */
                        return -EPERM;

                /*
                 * Doing anything requires privilege (go read about the
                 * "sendmail capabilities bug"), except for unprivileged bits.
                 * Indeed, the SECURE_ALL_UNPRIVILEGED bits are not
                 * restrictions enforced by the kernel but by user space on
                 * itself.
                 */
                if (cap_capable(current_cred(), current_cred()->user_ns,
                                CAP_SETPCAP, CAP_OPT_NONE) != 0) {
                        const unsigned long unpriv_and_locks =
                                SECURE_ALL_UNPRIVILEGED |
                                SECURE_ALL_UNPRIVILEGED << 1;
                        const unsigned long changed = old->securebits ^ arg2;

                        /* For legacy reason, denies non-change. */
                        if (!changed)
                                return -EPERM;

                        /* Denies privileged changes. */
                        if (changed & ~unpriv_and_locks)
                                return -EPERM;
                }

                new = prepare_creds();
                if (!new)
                        return -ENOMEM;
                new->securebits = arg2;
                return commit_creds(new);

        case PR_GET_SECUREBITS:
                return old->securebits;

        case PR_GET_KEEPCAPS:
                return !!issecure(SECURE_KEEP_CAPS);

        case PR_SET_KEEPCAPS:
                if (arg2 > 1) /* Note, we rely on arg2 being unsigned here */
                        return -EINVAL;
                if (issecure(SECURE_KEEP_CAPS_LOCKED))
                        return -EPERM;

                new = prepare_creds();
                if (!new)
                        return -ENOMEM;
                if (arg2)
                        new->securebits |= issecure_mask(SECURE_KEEP_CAPS);
                else
                        new->securebits &= ~issecure_mask(SECURE_KEEP_CAPS);
                return commit_creds(new);

        case PR_CAP_AMBIENT:
                if (arg2 == PR_CAP_AMBIENT_CLEAR_ALL) {
                        if (arg3 | arg4 | arg5)
                                return -EINVAL;

                        new = prepare_creds();
                        if (!new)
                                return -ENOMEM;
                        cap_clear(new->cap_ambient);
                        return commit_creds(new);
                }

                if (((!cap_valid(arg3)) | arg4 | arg5))
                        return -EINVAL;

                if (arg2 == PR_CAP_AMBIENT_IS_SET) {
                        return !!cap_raised(current_cred()->cap_ambient, arg3);
                } else if (arg2 != PR_CAP_AMBIENT_RAISE &&
                           arg2 != PR_CAP_AMBIENT_LOWER) {
                        return -EINVAL;
                } else {
                        if (arg2 == PR_CAP_AMBIENT_RAISE &&
                            (!cap_raised(current_cred()->cap_permitted, arg3) ||
                             !cap_raised(current_cred()->cap_inheritable,
                                         arg3) ||
                             issecure(SECURE_NO_CAP_AMBIENT_RAISE)))
                                return -EPERM;

                        new = prepare_creds();
                        if (!new)
                                return -ENOMEM;
                        if (arg2 == PR_CAP_AMBIENT_RAISE)
                                cap_raise(new->cap_ambient, arg3);
                        else
                                cap_lower(new->cap_ambient, arg3);
                        return commit_creds(new);
                }

        default:
                /* No functionality available - continue with default */
                return -ENOSYS;
        }
}

/**
 * cap_vm_enough_memory - Determine whether a new virtual mapping is permitted
 * @mm: The VM space in which the new mapping is to be made
 * @pages: The size of the mapping
 *
 * Determine whether the allocation of a new virtual mapping by the current
 * task is permitted.
 *
 * Return: 0 if permission granted, negative error code if not.
 */
int cap_vm_enough_memory(struct mm_struct *mm, long pages)
{
        return cap_capable(current_cred(), &init_user_ns, CAP_SYS_ADMIN,
                           CAP_OPT_NOAUDIT);
}

/**
 * cap_mmap_addr - check if able to map given addr
 * @addr: address attempting to be mapped
 *
 * If the process is attempting to map memory below dac_mmap_min_addr they need
 * CAP_SYS_RAWIO.  The other parameters to this function are unused by the
 * capability security module.
 *
 * Return: 0 if this mapping should be allowed or -EPERM if not.
 */
int cap_mmap_addr(unsigned long addr)
{
        int ret = 0;

        if (addr < dac_mmap_min_addr) {
                ret = cap_capable(current_cred(), &init_user_ns, CAP_SYS_RAWIO,
                                  CAP_OPT_NONE);
                /* set PF_SUPERPRIV if it turns out we allow the low mmap */
                if (ret == 0)
                        current->flags |= PF_SUPERPRIV;
        }
        return ret;
}

#ifdef CONFIG_SECURITY

static const struct lsm_id capability_lsmid = {
        .name = "capability",
        .id = LSM_ID_CAPABILITY,
};

static struct security_hook_list capability_hooks[] __ro_after_init = {
        LSM_HOOK_INIT(capable, cap_capable),
        LSM_HOOK_INIT(settime, cap_settime),
        LSM_HOOK_INIT(ptrace_access_check, cap_ptrace_access_check),
        LSM_HOOK_INIT(ptrace_traceme, cap_ptrace_traceme),
        LSM_HOOK_INIT(capget, cap_capget),
        LSM_HOOK_INIT(capset, cap_capset),
        LSM_HOOK_INIT(bprm_creds_from_file, cap_bprm_creds_from_file),
        LSM_HOOK_INIT(inode_need_killpriv, cap_inode_need_killpriv),
        LSM_HOOK_INIT(inode_killpriv, cap_inode_killpriv),
        LSM_HOOK_INIT(inode_getsecurity, cap_inode_getsecurity),
        LSM_HOOK_INIT(mmap_addr, cap_mmap_addr),
        LSM_HOOK_INIT(task_fix_setuid, cap_task_fix_setuid),
        LSM_HOOK_INIT(task_prctl, cap_task_prctl),
        LSM_HOOK_INIT(task_setscheduler, cap_task_setscheduler),
        LSM_HOOK_INIT(task_setioprio, cap_task_setioprio),
        LSM_HOOK_INIT(task_setnice, cap_task_setnice),
        LSM_HOOK_INIT(vm_enough_memory, cap_vm_enough_memory),
};

static int __init capability_init(void)
{
        security_add_hooks(capability_hooks, ARRAY_SIZE(capability_hooks),
                           &capability_lsmid);
        return 0;
}

DEFINE_LSM(capability) = {
        .id = &capability_lsmid,
        .order = LSM_ORDER_FIRST,
        .init = capability_init,
};

#endif /* CONFIG_SECURITY */

#ifdef CONFIG_SECURITY_COMMONCAP_KUNIT_TEST
#include "commoncap_test.c"
#endif


























    4 











    4 
    3 





    3 






















































    1 







    1 






    3 









    3 
    3 







































    4 
    3 

    4 
    4 




    1 













1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
// SPDX-License-Identifier: GPL-2.0-or-later
/* Validate the trust chain of a PKCS#7 message.
 *
 * Copyright (C) 2012 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#define pr_fmt(fmt) "PKCS7: "fmt
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/asn1.h>
#include <linux/key.h>
#include <keys/asymmetric-type.h>
#include <crypto/public_key.h>
#include "pkcs7_parser.h"

/*
 * Check the trust on one PKCS#7 SignedInfo block.
 */
static int pkcs7_validate_trust_one(struct pkcs7_message *pkcs7,
                                    struct pkcs7_signed_info *sinfo,
                                    struct key *trust_keyring)
{
        struct public_key_signature *sig = sinfo->sig;
        struct x509_certificate *x509, *last = NULL, *p;
        struct key *key;
        int ret;

        kenter(",%u,", sinfo->index);

        if (sinfo->unsupported_crypto) {
                kleave(" = -ENOPKG [cached]");
                return -ENOPKG;
        }

        for (x509 = sinfo->signer; x509; x509 = x509->signer) {
                if (x509->seen) {
                        if (x509->verified)
                                goto verified;
                        kleave(" = -ENOKEY [cached]");
                        return -ENOKEY;
                }
                x509->seen = true;

                /* Look to see if this certificate is present in the trusted
                 * keys.
                 */
                key = find_asymmetric_key(trust_keyring,
                                          x509->id, x509->skid, NULL, false);
                if (!IS_ERR(key)) {
                        /* One of the X.509 certificates in the PKCS#7 message
                         * is apparently the same as one we already trust.
                         * Verify that the trusted variant can also validate
                         * the signature on the descendant.
                         */
                        pr_devel("sinfo %u: Cert %u as key %x\n",
                                 sinfo->index, x509->index, key_serial(key));
                        goto matched;
                }
                if (key == ERR_PTR(-ENOMEM))
                        return -ENOMEM;

                 /* Self-signed certificates form roots of their own, and if we
                  * don't know them, then we can't accept them.
                  */
                if (x509->signer == x509) {
                        kleave(" = -ENOKEY [unknown self-signed]");
                        return -ENOKEY;
                }

                might_sleep();
                last = x509;
                sig = last->sig;
        }

        /* No match - see if the root certificate has a signer amongst the
         * trusted keys.
         */
        if (last && (last->sig->auth_ids[0] || last->sig->auth_ids[1])) {
                key = find_asymmetric_key(trust_keyring,
                                          last->sig->auth_ids[0],
                                          last->sig->auth_ids[1],
                                          NULL, false);
                if (!IS_ERR(key)) {
                        x509 = last;
                        pr_devel("sinfo %u: Root cert %u signer is key %x\n",
                                 sinfo->index, x509->index, key_serial(key));
                        goto matched;
                }
                if (PTR_ERR(key) != -ENOKEY)
                        return PTR_ERR(key);
        }

        /* As a last resort, see if we have a trusted public key that matches
         * the signed info directly.
         */
        key = find_asymmetric_key(trust_keyring,
                                  sinfo->sig->auth_ids[0], NULL, NULL, false);
        if (!IS_ERR(key)) {
                pr_devel("sinfo %u: Direct signer is key %x\n",
                         sinfo->index, key_serial(key));
                x509 = NULL;
                sig = sinfo->sig;
                goto matched;
        }
        if (PTR_ERR(key) != -ENOKEY)
                return PTR_ERR(key);

        kleave(" = -ENOKEY [no backref]");
        return -ENOKEY;

matched:
        ret = verify_signature(key, sig);
        key_put(key);
        if (ret < 0) {
                if (ret == -ENOMEM)
                        return ret;
                kleave(" = -EKEYREJECTED [verify %d]", ret);
                return -EKEYREJECTED;
        }

verified:
        if (x509) {
                x509->verified = true;
                for (p = sinfo->signer; p != x509; p = p->signer)
                        p->verified = true;
        }
        kleave(" = 0");
        return 0;
}

/**
 * pkcs7_validate_trust - Validate PKCS#7 trust chain
 * @pkcs7: The PKCS#7 certificate to validate
 * @trust_keyring: Signing certificates to use as starting points
 *
 * Validate that the certificate chain inside the PKCS#7 message intersects
 * keys we already know and trust.
 *
 * Returns, in order of descending priority:
 *
 *  (*) -EKEYREJECTED if a signature failed to match for which we have a valid
 *        key, or:
 *
 *  (*) 0 if at least one signature chain intersects with the keys in the trust
 *        keyring, or:
 *
 *  (*) -ENOPKG if a suitable crypto module couldn't be found for a check on a
 *        chain.
 *
 *  (*) -ENOKEY if we couldn't find a match for any of the signature chains in
 *        the message.
 *
 * May also return -ENOMEM.
 */
int pkcs7_validate_trust(struct pkcs7_message *pkcs7,
                         struct key *trust_keyring)
{
        struct pkcs7_signed_info *sinfo;
        struct x509_certificate *p;
        int cached_ret = -ENOKEY;
        int ret;

        for (p = pkcs7->certs; p; p = p->next)
                p->seen = false;

        for (sinfo = pkcs7->signed_infos; sinfo; sinfo = sinfo->next) {
                ret = pkcs7_validate_trust_one(pkcs7, sinfo, trust_keyring);
                switch (ret) {
                case -ENOKEY:
                        continue;
                case -ENOPKG:
                        if (cached_ret == -ENOKEY)
                                cached_ret = -ENOPKG;
                        continue;
                case 0:
                        cached_ret = 0;
                        continue;
                default:
                        return ret;
                }
        }

        return cached_ret;
}
EXPORT_SYMBOL_GPL(pkcs7_validate_trust);

























































    2 
    2 



    2 



    2 



    2 

















    2 


   17 
   17 












   61 























































































    1 

    1 


    1 
    1 



























   61 
   61 

   61 



   61 










   61 





   61 

   53 






   18 

   18 
   18 


   16 
    8 
    4 




    2 



   59 


   61 









    3 














   62 








   62 





   62 


   62 






   62 


   19 


   53 


   62 




    5 


    5 
    3 


    1 



    2 

    1 





















   61 
   18 

   61 













   59 











   55 







   61 
    3 

   61 





   61 
    2 

    1 

    2 



   61 
   62 






































































































































































   13 






   59 











    1 

    1 










    1 
    1 
    1 


   19 






   19 









    4 

   62 


    4 








    6 


















    4 

    3 


    4 












    4 







    4 

    4 


    1 

   12 















    9 







    9 




    9 













    9 






    8 




    9 


    8 


    8 

    8 










    7 
    7 
    6 

    5 


    1 
    1 
    1 










    3 







    3 

    1 






    3 
    3 










    1 

    2 


    2 



    2 
    1 





    2 










    4 














    4 



    4 












    4 










    3 


    1 



    3 


    4 

    4 


    3 


    4 



    4 



    6 







   12 
   12 

    9 








    6 
    3 

    6 

















    7 




    1 



    6 

    7 














   26 












   11 




























   23 















   32 



   32 







   32 
    1 

   32 





   32 






    3 


    1 

    3 










    9 












    6 

    6 
    3 
    3 

    6 






   16 



    8 






   12 










   12 






   20 



    8 






   20 









   16 





   16 





   10 


















   32 











   12 




























   13 







    7 





    7 



    7 






    3 













    7 
































































   10 




    1 
   10 
























































    9 

   13 
    8 



   11 
    7 



    9 
    9 

    9 


    9 



    9 
    9 











   13 
   13 


   10 


   13 



















































































































































































































































































































































































































































    3 


    2 
    2 


    2 





    3 

    2 

    3 







































    2 











    1 

    2 

    2 














1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
// SPDX-License-Identifier: GPL-2.0-only
/*
 * common code for virtio vsock
 *
 * Copyright (C) 2013-2015 Red Hat, Inc.
 * Author: Asias He <asias@redhat.com>
 *         Stefan Hajnoczi <stefanha@redhat.com>
 */
#include <linux/spinlock.h>
#include <linux/module.h>
#include <linux/sched/signal.h>
#include <linux/ctype.h>
#include <linux/list.h>
#include <linux/virtio_vsock.h>
#include <uapi/linux/vsockmon.h>

#include <net/sock.h>
#include <net/af_vsock.h>

#define CREATE_TRACE_POINTS
#include <trace/events/vsock_virtio_transport_common.h>

/* How long to wait for graceful shutdown of a connection */
#define VSOCK_CLOSE_TIMEOUT (8 * HZ)

/* Threshold for detecting small packets to copy */
#define GOOD_COPY_LEN  128

/* Max payload that can be collapsed into a single linear skb, using the same
 * allocation threshold as virtio_vsock_alloc_skb() to avoid adding pressure
 * on the page allocator.
 */
#define MAX_COLLAPSE_LEN \
        SKB_MAX_ORDER(VIRTIO_VSOCK_SKB_HEADROOM, PAGE_ALLOC_COSTLY_ORDER)

static void virtio_transport_cancel_close_work(struct vsock_sock *vsk,
                                               bool cancel_timeout);
static s64 virtio_transport_has_space(struct virtio_vsock_sock *vvs);

static const struct virtio_transport *
virtio_transport_get_ops(struct vsock_sock *vsk)
{
        const struct vsock_transport *t = vsock_core_get_transport(vsk);

        if (WARN_ON(!t))
                return NULL;

        return container_of(t, struct virtio_transport, transport);
}

static bool virtio_transport_can_zcopy(const struct virtio_transport *t_ops,
                                       struct virtio_vsock_pkt_info *info,
                                       size_t pkt_len)
{
        struct iov_iter *iov_iter;

        if (!info->msg)
                return false;

        iov_iter = &info->msg->msg_iter;

        if (iov_iter->iov_offset)
                return false;

        /* We can't send whole iov. */
        if (iov_iter->count > pkt_len)
                return false;

        /* Check that transport can send data in zerocopy mode. */
        if (t_ops->can_msgzerocopy) {
                int pages_to_send = iov_iter_npages(iov_iter, MAX_SKB_FRAGS);

                /* +1 is for packet header. */
                return t_ops->can_msgzerocopy(pages_to_send + 1);
        }

        return true;
}

static int virtio_transport_fill_skb(struct sk_buff *skb,
                                     struct virtio_vsock_pkt_info *info,
                                     size_t len,
                                     bool zcopy)
{
        struct msghdr *msg = info->msg;

        if (zcopy)
                return __zerocopy_sg_from_iter(msg, NULL, skb,
                                               &msg->msg_iter, len, NULL);

        virtio_vsock_skb_put(skb, len);
        return skb_copy_datagram_from_iter_full(skb, 0, &msg->msg_iter, len);
}

static void virtio_transport_init_hdr(struct sk_buff *skb,
                                      struct virtio_vsock_pkt_info *info,
                                      size_t payload_len,
                                      u32 src_cid,
                                      u32 src_port,
                                      u32 dst_cid,
                                      u32 dst_port)
{
        struct virtio_vsock_hdr *hdr;

        hdr = virtio_vsock_hdr(skb);
        hdr->type        = cpu_to_le16(info->type);
        hdr->op                = cpu_to_le16(info->op);
        hdr->src_cid        = cpu_to_le64(src_cid);
        hdr->dst_cid        = cpu_to_le64(dst_cid);
        hdr->src_port        = cpu_to_le32(src_port);
        hdr->dst_port        = cpu_to_le32(dst_port);
        hdr->flags        = cpu_to_le32(info->flags);
        hdr->len        = cpu_to_le32(payload_len);
        hdr->buf_alloc        = cpu_to_le32(0);
        hdr->fwd_cnt        = cpu_to_le32(0);
}

/* Packet capture */
static struct sk_buff *virtio_transport_build_skb(void *opaque)
{
        struct virtio_vsock_hdr *pkt_hdr;
        struct sk_buff *pkt = opaque;
        struct af_vsockmon_hdr *hdr;
        struct sk_buff *skb;
        size_t payload_len;

        /* A packet could be split to fit the RX buffer, so we use
         * the payload length from the header, which has been updated
         * by the sender to reflect the fragment size.
         */
        pkt_hdr = virtio_vsock_hdr(pkt);
        payload_len = le32_to_cpu(pkt_hdr->len);

        skb = alloc_skb(sizeof(*hdr) + sizeof(*pkt_hdr) + payload_len,
                        GFP_ATOMIC);
        if (!skb)
                return NULL;

        hdr = skb_put(skb, sizeof(*hdr));

        /* pkt->hdr is little-endian so no need to byteswap here */
        hdr->src_cid = pkt_hdr->src_cid;
        hdr->src_port = pkt_hdr->src_port;
        hdr->dst_cid = pkt_hdr->dst_cid;
        hdr->dst_port = pkt_hdr->dst_port;

        hdr->transport = cpu_to_le16(AF_VSOCK_TRANSPORT_VIRTIO);
        hdr->len = cpu_to_le16(sizeof(*pkt_hdr));
        memset(hdr->reserved, 0, sizeof(hdr->reserved));

        switch (le16_to_cpu(pkt_hdr->op)) {
        case VIRTIO_VSOCK_OP_REQUEST:
        case VIRTIO_VSOCK_OP_RESPONSE:
                hdr->op = cpu_to_le16(AF_VSOCK_OP_CONNECT);
                break;
        case VIRTIO_VSOCK_OP_RST:
        case VIRTIO_VSOCK_OP_SHUTDOWN:
                hdr->op = cpu_to_le16(AF_VSOCK_OP_DISCONNECT);
                break;
        case VIRTIO_VSOCK_OP_RW:
                hdr->op = cpu_to_le16(AF_VSOCK_OP_PAYLOAD);
                break;
        case VIRTIO_VSOCK_OP_CREDIT_UPDATE:
        case VIRTIO_VSOCK_OP_CREDIT_REQUEST:
                hdr->op = cpu_to_le16(AF_VSOCK_OP_CONTROL);
                break;
        default:
                hdr->op = cpu_to_le16(AF_VSOCK_OP_UNKNOWN);
                break;
        }

        skb_put_data(skb, pkt_hdr, sizeof(*pkt_hdr));

        if (payload_len) {
                struct iov_iter iov_iter;
                struct kvec kvec;
                void *data = skb_put(skb, payload_len);

                kvec.iov_base = data;
                kvec.iov_len = payload_len;
                iov_iter_kvec(&iov_iter, ITER_DEST, &kvec, 1, payload_len);

                if (skb_copy_datagram_iter(pkt, VIRTIO_VSOCK_SKB_CB(pkt)->offset,
                                           &iov_iter, payload_len)) {
                        kfree_skb(skb);
                        return NULL;
                }
        }

        return skb;
}

void virtio_transport_deliver_tap_pkt(struct sk_buff *skb)
{
        if (virtio_vsock_skb_tap_delivered(skb))
                return;

        vsock_deliver_tap(virtio_transport_build_skb, skb);
        virtio_vsock_skb_set_tap_delivered(skb);
}
EXPORT_SYMBOL_GPL(virtio_transport_deliver_tap_pkt);

static u16 virtio_transport_get_type(struct sock *sk)
{
        if (sk->sk_type == SOCK_STREAM)
                return VIRTIO_VSOCK_TYPE_STREAM;
        else
                return VIRTIO_VSOCK_TYPE_SEQPACKET;
}

/* Returns new sk_buff on success, otherwise returns NULL. */
static struct sk_buff *virtio_transport_alloc_skb(struct virtio_vsock_pkt_info *info,
                                                  size_t payload_len,
                                                  bool zcopy,
                                                  struct ubuf_info *uarg,
                                                  u32 src_cid,
                                                  u32 src_port,
                                                  u32 dst_cid,
                                                  u32 dst_port)
{
        struct vsock_sock *vsk;
        struct sk_buff *skb;
        size_t skb_len;

        skb_len = VIRTIO_VSOCK_SKB_HEADROOM;

        if (!zcopy)
                skb_len += payload_len;

        skb = virtio_vsock_alloc_skb(skb_len, GFP_KERNEL);
        if (!skb)
                return NULL;

        virtio_transport_init_hdr(skb, info, payload_len, src_cid, src_port,
                                  dst_cid, dst_port);

        vsk = info->vsk;

        /* If 'vsk' != NULL then payload is always present, so we
         * will never call '__zerocopy_sg_from_iter()' below without
         * setting skb owner in 'skb_set_owner_w()'. The only case
         * when 'vsk' == NULL is VIRTIO_VSOCK_OP_RST control message
         * without payload.
         */
        WARN_ON_ONCE(!(vsk && (info->msg && payload_len)) && zcopy);

        /* Set owner here, because '__zerocopy_sg_from_iter()' uses
         * owner of skb without check to update 'sk_wmem_alloc'.
         */
        if (vsk)
                skb_set_owner_w(skb, sk_vsock(vsk));

        if (info->msg && payload_len > 0) {
                int err;

                /* Bind the zerocopy lifetime before filling frags so error
                 * rollback frees managed fixed-buffer pages through
                 * the uarg-aware path.
                 */
                skb_zcopy_set(skb, uarg, NULL);

                err = virtio_transport_fill_skb(skb, info, payload_len, zcopy);
                if (err)
                        goto out;

                if (msg_data_left(info->msg) == 0 &&
                    info->type == VIRTIO_VSOCK_TYPE_SEQPACKET) {
                        struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);

                        hdr->flags |= cpu_to_le32(VIRTIO_VSOCK_SEQ_EOM);

                        if (info->msg->msg_flags & MSG_EOR)
                                hdr->flags |= cpu_to_le32(VIRTIO_VSOCK_SEQ_EOR);
                }
        }

        if (info->reply)
                virtio_vsock_skb_set_reply(skb);

        trace_virtio_transport_alloc_pkt(src_cid, src_port,
                                         dst_cid, dst_port,
                                         payload_len,
                                         info->type,
                                         info->op,
                                         info->flags,
                                         zcopy);

        return skb;
out:
        kfree_skb(skb);
        return NULL;
}

/* This function can only be used on connecting/connected sockets,
 * since a socket assigned to a transport is required.
 *
 * Do not use on listener sockets!
 */
static int virtio_transport_send_pkt_info(struct vsock_sock *vsk,
                                          struct virtio_vsock_pkt_info *info)
{
        u32 max_skb_len = VIRTIO_VSOCK_MAX_PKT_BUF_SIZE;
        u32 src_cid, src_port, dst_cid, dst_port;
        const struct virtio_transport *t_ops;
        struct iov_iter_state msg_iter_state;
        struct virtio_vsock_sock *vvs;
        struct ubuf_info *uarg = NULL;
        u32 pkt_len = info->pkt_len;
        bool can_zcopy = false;
        bool have_uref = false;
        u32 rest_len;
        int ret;

        info->type = virtio_transport_get_type(sk_vsock(vsk));

        t_ops = virtio_transport_get_ops(vsk);
        if (unlikely(!t_ops))
                return -EFAULT;

        src_cid = t_ops->transport.get_local_cid();
        src_port = vsk->local_addr.svm_port;
        if (!info->remote_cid) {
                dst_cid        = vsk->remote_addr.svm_cid;
                dst_port = vsk->remote_addr.svm_port;
        } else {
                dst_cid = info->remote_cid;
                dst_port = info->remote_port;
        }

        vvs = vsk->trans;

        /* virtio_transport_get_credit might return less than pkt_len credit */
        pkt_len = virtio_transport_get_credit(vvs, pkt_len);

        /* Do not send zero length OP_RW pkt */
        if (pkt_len == 0 && info->op == VIRTIO_VSOCK_OP_RW)
                return pkt_len;

        if (info->msg && (info->msg->msg_flags & MSG_ZEROCOPY)) {
                /* If 'info->msg' is not NULL, this is only VIRTIO_VSOCK_OP_RW.
                 * 'MSG_ZEROCOPY' flag handling here is based on the same flag
                 * handling from 'tcp_sendmsg_locked()'.
                 */
                if (info->msg->msg_ubuf) {
                        uarg = info->msg->msg_ubuf;
                        can_zcopy = virtio_transport_can_zcopy(t_ops, info, pkt_len);
                } else if (sock_flag(sk_vsock(vsk), SOCK_ZEROCOPY)) {
                        uarg = msg_zerocopy_realloc(sk_vsock(vsk), pkt_len,
                                                    NULL, false);
                        if (!uarg) {
                                virtio_transport_put_credit(vvs, pkt_len);
                                return -ENOMEM;
                        }

                        can_zcopy = virtio_transport_can_zcopy(t_ops, info, pkt_len);
                        if (!can_zcopy)
                                uarg_to_msgzc(uarg)->zerocopy = 0;

                        have_uref = true;
                }

                /* 'can_zcopy' means that this transmission will be
                 * in zerocopy way (e.g. using 'frags' array).
                 */
                if (can_zcopy)
                        max_skb_len = min_t(u32, VIRTIO_VSOCK_MAX_PKT_BUF_SIZE,
                                            (MAX_SKB_FRAGS * PAGE_SIZE));
        }

        rest_len = pkt_len;

        do {
                struct sk_buff *skb;
                size_t skb_len;

                /* Save iterator state in case allocation or transmission fails
                 * so we can restore it and retry.
                 */
                if (info->msg)
                        iov_iter_save_state(&info->msg->msg_iter, &msg_iter_state);

                skb_len = min(max_skb_len, rest_len);

                /* Note: virtio_transport_alloc_skb() can advance info->msg->msg_iter
                 * even if it fails (e.g. partial GUP success).
                 */
                skb = virtio_transport_alloc_skb(info, skb_len, can_zcopy,
                                                 uarg,
                                                 src_cid, src_port,
                                                 dst_cid, dst_port);
                if (!skb) {
                        ret = -ENOMEM;
                        break;
                }

                virtio_transport_inc_tx_pkt(vvs, skb);

                ret = t_ops->send_pkt(skb, info->net);
                if (ret < 0)
                        break;

                /* Both virtio and vhost 'send_pkt()' returns 'skb_len',
                 * but for reliability use 'ret' instead of 'skb_len'.
                 * Also if partial send happens (e.g. 'ret' != 'skb_len')
                 * somehow, we break this loop, but account such returned
                 * value in 'virtio_transport_put_credit()'.
                 */
                rest_len -= ret;

                if (WARN_ONCE(ret != skb_len,
                              "'send_pkt()' returns %i, but %zu expected\n",
                              ret, skb_len))
                        break;
        } while (rest_len);

        if (info->msg && ret < 0)
                iov_iter_restore(&info->msg->msg_iter, &msg_iter_state);

        virtio_transport_put_credit(vvs, rest_len);

        /* msg_zerocopy_realloc() initializes the ubuf_info refcnt to 1.
         * skb_zcopy_set() increases it for each skb, so we can drop that
         * initial reference to keep it balanced.
         */
        if (have_uref) {
                if (rest_len == pkt_len)
                        /* No data sent, abort the notification. */
                        net_zcopy_put_abort(uarg, true);
                else
                        net_zcopy_put(uarg);
        }

        /* Return number of bytes, if any data has been sent. */
        if (rest_len != pkt_len)
                ret = pkt_len - rest_len;

        return ret;
}

static bool virtio_transport_can_collapse(struct sk_buff *skb)
{
        /* skbs that are partially consumed, mark a SEQPACKET message boundary,
         * or are already large enough should not be collapsed: they either
         * need special accounting, carry protocol state, or already have a
         * good data-to-overhead ratio.
         */
        if (VIRTIO_VSOCK_SKB_CB(skb)->offset)
                return false;
        if (le32_to_cpu(virtio_vsock_hdr(skb)->flags) & VIRTIO_VSOCK_SEQ_EOM)
                return false;
        if (skb->len >= MAX_COLLAPSE_LEN)
                return false;
        return true;
}

/* Iterate through the packets in the queue starting from the current skb to
 * count the number of bytes we can collapse.
 */
static unsigned int
virtio_transport_collapse_size(struct sk_buff *skb, struct sk_buff_head *queue)
{
        unsigned int target = skb->len - VIRTIO_VSOCK_SKB_CB(skb)->offset;

        while ((skb = skb_peek_next(skb, queue)) &&
               virtio_transport_can_collapse(skb)) {
                unsigned int len = skb->len - VIRTIO_VSOCK_SKB_CB(skb)->offset;

                if (len > MAX_COLLAPSE_LEN - target)
                        return target;

                target += len;
        }

        return target;
}

/* Called under lock_sock to compact the receive queue by merging small skbs.
 * @min_to_free: minimum number of skbs to eliminate from the queue. May free
 *               more to fill each collapsed skb to capacity.
 */
static void
virtio_transport_collapse_rx_queue(struct virtio_vsock_sock *vvs,
                                   u32 min_to_free)
{
        struct sk_buff *skb, *next_skb, *new_skb = NULL;
        struct sk_buff_head new_queue;
        u32 saved = 0;

        __skb_queue_head_init(&new_queue);

        skb_queue_walk_safe(&vvs->rx_queue, skb, next_skb) {
                struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
                u32 src_off = VIRTIO_VSOCK_SKB_CB(skb)->offset;
                u32 src_len = skb->len - src_off;
                bool keep;

                keep = !virtio_transport_can_collapse(skb);
                if (keep) {
                        /* Finalize pending collapsed skb to preserve packet
                         * ordering.
                         */
                        if (new_skb) {
                                __skb_queue_tail(&new_queue, new_skb);
                                new_skb = NULL;
                                saved--;
                        }
                        goto next;
                }

                /* Finalize if this packet won't fit in the remaining tailroom,
                 * so we can allocate a right-sized new_skb.
                 */
                if (new_skb && src_len > skb_tailroom(new_skb)) {
                        __skb_queue_tail(&new_queue, new_skb);
                        new_skb = NULL;
                        saved--;
                }

                if (!new_skb) {
                        unsigned int alloc_size;

                        /* Check after finalizing to opportunistically fill
                         * each collapsed skb to capacity, merging more skbs
                         * than strictly required.
                         */
                        if (saved >= min_to_free)
                                break;

                        alloc_size = virtio_transport_collapse_size(skb, &vvs->rx_queue);

                        /* Only this skb's data is eligible, nothing to merge
                         * with. Keep as-is.
                         */
                        if (alloc_size <= src_len) {
                                keep = true;
                                goto next;
                        }

                        new_skb = virtio_vsock_alloc_linear_skb(alloc_size +
                                        VIRTIO_VSOCK_SKB_HEADROOM, GFP_KERNEL);
                        if (!new_skb)
                                break;

                        memcpy(virtio_vsock_hdr(new_skb), hdr,
                               sizeof(struct virtio_vsock_hdr));
                        virtio_vsock_hdr(new_skb)->len = 0;
                }

                /* Cannot fail since src_off/src_len are within bounds, but if
                 * it does, discard new_skb to avoid queuing corrupted data.
                 */
                if (WARN_ON_ONCE(skb_copy_bits(skb, src_off,
                                               skb_put(new_skb, src_len),
                                               src_len))) {
                        kfree_skb(new_skb);
                        new_skb = NULL;
                        break;
                }

                le32_add_cpu(&virtio_vsock_hdr(new_skb)->len, src_len);
                virtio_vsock_hdr(new_skb)->flags |= hdr->flags;

next:
                __skb_unlink(skb, &vvs->rx_queue);
                if (keep) {
                        __skb_queue_tail(&new_queue, skb);
                } else {
                        consume_skb(skb);
                        saved++;
                }
        }

        if (new_skb)
                __skb_queue_tail(&new_queue, new_skb);

        skb_queue_splice(&new_queue, &vvs->rx_queue);
}

static bool virtio_transport_inc_rx_pkt(struct virtio_vsock_sock *vvs,
                                        u32 len)
{
        u64 skb_overhead = (skb_queue_len(&vvs->rx_queue) + 1) * SKB_TRUESIZE(0);

        /* Allow at most buf_alloc * 2 total budget (payload + overhead),
         * similar to how SO_RCVBUF is doubled to reserve space for sk_buff
         * metadata. Check payload against buf_alloc to be sure the other
         * peer is respecting the credit, and sk_buff overhead to bound
         * queue growth.
         */
        if ((u64)vvs->buf_used + len > vvs->buf_alloc ||
            skb_overhead > vvs->buf_alloc)
                return false;

        vvs->rx_bytes += len;
        vvs->buf_used += len;
        return true;
}

static void virtio_transport_dec_rx_pkt(struct virtio_vsock_sock *vvs,
                                        u32 bytes_read, u32 bytes_dequeued)
{
        vvs->rx_bytes -= bytes_read;
        vvs->buf_used -= bytes_dequeued;
        vvs->fwd_cnt += bytes_dequeued;
}

void virtio_transport_inc_tx_pkt(struct virtio_vsock_sock *vvs, struct sk_buff *skb)
{
        struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);

        spin_lock_bh(&vvs->rx_lock);
        vvs->last_fwd_cnt = vvs->fwd_cnt;
        hdr->fwd_cnt = cpu_to_le32(vvs->fwd_cnt);
        hdr->buf_alloc = cpu_to_le32(vvs->buf_alloc);
        spin_unlock_bh(&vvs->rx_lock);
}
EXPORT_SYMBOL_GPL(virtio_transport_inc_tx_pkt);

void virtio_transport_consume_skb_sent(struct sk_buff *skb, bool consume)
{
        struct sock *s = skb->sk;

        if (s && skb->len) {
                struct vsock_sock *vs = vsock_sk(s);
                struct virtio_vsock_sock *vvs;

                vvs = vs->trans;

                spin_lock_bh(&vvs->tx_lock);
                vvs->bytes_unsent -= skb->len;
                spin_unlock_bh(&vvs->tx_lock);
        }

        if (consume)
                consume_skb(skb);
}
EXPORT_SYMBOL_GPL(virtio_transport_consume_skb_sent);

u32 virtio_transport_get_credit(struct virtio_vsock_sock *vvs, u32 credit)
{
        u32 ret;

        if (!credit)
                return 0;

        spin_lock_bh(&vvs->tx_lock);
        ret = min_t(u32, credit, virtio_transport_has_space(vvs));
        vvs->tx_cnt += ret;
        vvs->bytes_unsent += ret;
        spin_unlock_bh(&vvs->tx_lock);

        return ret;
}
EXPORT_SYMBOL_GPL(virtio_transport_get_credit);

void virtio_transport_put_credit(struct virtio_vsock_sock *vvs, u32 credit)
{
        if (!credit)
                return;

        spin_lock_bh(&vvs->tx_lock);
        vvs->tx_cnt -= credit;
        vvs->bytes_unsent -= credit;
        spin_unlock_bh(&vvs->tx_lock);
}
EXPORT_SYMBOL_GPL(virtio_transport_put_credit);

static int virtio_transport_send_credit_update(struct vsock_sock *vsk)
{
        struct virtio_vsock_pkt_info info = {
                .op = VIRTIO_VSOCK_OP_CREDIT_UPDATE,
                .vsk = vsk,
                .net = sock_net(sk_vsock(vsk)),
        };

        return virtio_transport_send_pkt_info(vsk, &info);
}

static ssize_t
virtio_transport_stream_do_peek(struct vsock_sock *vsk,
                                struct msghdr *msg,
                                size_t len)
{
        struct virtio_vsock_sock *vvs = vsk->trans;
        struct sk_buff *skb;
        size_t total = 0;
        int err;

        spin_lock_bh(&vvs->rx_lock);

        skb_queue_walk(&vvs->rx_queue, skb) {
                size_t bytes;

                bytes = min_t(size_t, len - total,
                              skb->len - VIRTIO_VSOCK_SKB_CB(skb)->offset);

                spin_unlock_bh(&vvs->rx_lock);

                /* sk_lock is held by caller so no one else can dequeue.
                 * Unlock rx_lock since skb_copy_datagram_iter() may sleep.
                 */
                err = skb_copy_datagram_iter(skb, VIRTIO_VSOCK_SKB_CB(skb)->offset,
                                             &msg->msg_iter, bytes);
                if (err)
                        goto out;

                total += bytes;

                spin_lock_bh(&vvs->rx_lock);

                if (total == len)
                        break;
        }

        spin_unlock_bh(&vvs->rx_lock);

        return total;

out:
        if (total)
                err = total;
        return err;
}

static ssize_t
virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
                                   struct msghdr *msg,
                                   size_t len)
{
        struct virtio_vsock_sock *vvs = vsk->trans;
        struct sk_buff *skb;
        u32 fwd_cnt_delta;
        bool low_rx_bytes;
        int err = -EFAULT;
        size_t total = 0;
        u32 free_space;

        spin_lock_bh(&vvs->rx_lock);

        if (WARN_ONCE(skb_queue_empty(&vvs->rx_queue) && vvs->rx_bytes,
                      "rx_queue is empty, but rx_bytes is non-zero\n")) {
                spin_unlock_bh(&vvs->rx_lock);
                return err;
        }

        while (total < len && !skb_queue_empty(&vvs->rx_queue)) {
                size_t bytes, dequeued = 0;

                skb = skb_peek(&vvs->rx_queue);

                bytes = min_t(size_t, len - total,
                              skb->len - VIRTIO_VSOCK_SKB_CB(skb)->offset);

                /* sk_lock is held by caller so no one else can dequeue.
                 * Unlock rx_lock since skb_copy_datagram_iter() may sleep.
                 */
                spin_unlock_bh(&vvs->rx_lock);

                err = skb_copy_datagram_iter(skb,
                                             VIRTIO_VSOCK_SKB_CB(skb)->offset,
                                             &msg->msg_iter, bytes);
                if (err)
                        goto out;

                spin_lock_bh(&vvs->rx_lock);

                total += bytes;

                VIRTIO_VSOCK_SKB_CB(skb)->offset += bytes;

                if (skb->len == VIRTIO_VSOCK_SKB_CB(skb)->offset) {
                        dequeued = le32_to_cpu(virtio_vsock_hdr(skb)->len);
                        __skb_unlink(skb, &vvs->rx_queue);
                        consume_skb(skb);
                }

                virtio_transport_dec_rx_pkt(vvs, bytes, dequeued);
        }

        fwd_cnt_delta = vvs->fwd_cnt - vvs->last_fwd_cnt;
        free_space = vvs->buf_alloc - fwd_cnt_delta;
        low_rx_bytes = (vvs->rx_bytes <
                        sock_rcvlowat(sk_vsock(vsk), 0, INT_MAX));

        spin_unlock_bh(&vvs->rx_lock);

        /* To reduce the number of credit update messages,
         * don't update credits as long as lots of space is available.
         * Note: the limit chosen here is arbitrary. Setting the limit
         * too high causes extra messages. Too low causes transmitter
         * stalls. As stalls are in theory more expensive than extra
         * messages, we set the limit to a high value. TODO: experiment
         * with different values. Also send credit update message when
         * number of bytes in rx queue is not enough to wake up reader.
         */
        if (fwd_cnt_delta &&
            (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE || low_rx_bytes))
                virtio_transport_send_credit_update(vsk);

        return total;

out:
        if (total)
                err = total;
        return err;
}

static ssize_t
virtio_transport_seqpacket_do_peek(struct vsock_sock *vsk,
                                   struct msghdr *msg)
{
        struct virtio_vsock_sock *vvs = vsk->trans;
        struct sk_buff *skb;
        size_t total, len;

        spin_lock_bh(&vvs->rx_lock);

        if (!vvs->msg_count) {
                spin_unlock_bh(&vvs->rx_lock);
                return 0;
        }

        total = 0;
        len = msg_data_left(msg);

        skb_queue_walk(&vvs->rx_queue, skb) {
                struct virtio_vsock_hdr *hdr;

                if (total < len) {
                        size_t bytes;
                        int err;

                        bytes = len - total;
                        if (bytes > skb->len)
                                bytes = skb->len;

                        spin_unlock_bh(&vvs->rx_lock);

                        /* sk_lock is held by caller so no one else can dequeue.
                         * Unlock rx_lock since skb_copy_datagram_iter() may sleep.
                         */
                        err = skb_copy_datagram_iter(skb, VIRTIO_VSOCK_SKB_CB(skb)->offset,
                                                     &msg->msg_iter, bytes);
                        if (err)
                                return err;

                        spin_lock_bh(&vvs->rx_lock);
                }

                total += skb->len;
                hdr = virtio_vsock_hdr(skb);

                if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) {
                        if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOR)
                                msg->msg_flags |= MSG_EOR;

                        break;
                }
        }

        spin_unlock_bh(&vvs->rx_lock);

        return total;
}

static int virtio_transport_seqpacket_do_dequeue(struct vsock_sock *vsk,
                                                 struct msghdr *msg,
                                                 int flags)
{
        struct virtio_vsock_sock *vvs = vsk->trans;
        int dequeued_len = 0;
        size_t user_buf_len = msg_data_left(msg);
        bool msg_ready = false;
        struct sk_buff *skb;

        spin_lock_bh(&vvs->rx_lock);

        if (vvs->msg_count == 0) {
                spin_unlock_bh(&vvs->rx_lock);
                return 0;
        }

        while (!msg_ready) {
                struct virtio_vsock_hdr *hdr;
                size_t pkt_len;

                skb = __skb_dequeue(&vvs->rx_queue);
                if (!skb)
                        break;
                hdr = virtio_vsock_hdr(skb);
                pkt_len = (size_t)le32_to_cpu(hdr->len);

                if (dequeued_len >= 0) {
                        size_t bytes_to_copy;

                        bytes_to_copy = min(user_buf_len, pkt_len);

                        if (bytes_to_copy) {
                                int err;

                                /* sk_lock is held by caller so no one else can dequeue.
                                 * Unlock rx_lock since skb_copy_datagram_iter() may sleep.
                                 */
                                spin_unlock_bh(&vvs->rx_lock);

                                err = skb_copy_datagram_iter(skb, 0,
                                                             &msg->msg_iter,
                                                             bytes_to_copy);
                                if (err) {
                                        /* Copy of message failed. Rest of
                                         * fragments will be freed without copy.
                                         */
                                        dequeued_len = err;
                                } else {
                                        user_buf_len -= bytes_to_copy;
                                }

                                spin_lock_bh(&vvs->rx_lock);
                        }

                        if (dequeued_len >= 0)
                                dequeued_len += pkt_len;
                }

                if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) {
                        msg_ready = true;
                        vvs->msg_count--;

                        if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOR)
                                msg->msg_flags |= MSG_EOR;
                }

                virtio_transport_dec_rx_pkt(vvs, pkt_len, pkt_len);
                kfree_skb(skb);
        }

        spin_unlock_bh(&vvs->rx_lock);

        virtio_transport_send_credit_update(vsk);

        return dequeued_len;
}

ssize_t
virtio_transport_stream_dequeue(struct vsock_sock *vsk,
                                struct msghdr *msg,
                                size_t len, int flags)
{
        if (flags & MSG_PEEK)
                return virtio_transport_stream_do_peek(vsk, msg, len);
        else
                return virtio_transport_stream_do_dequeue(vsk, msg, len);
}
EXPORT_SYMBOL_GPL(virtio_transport_stream_dequeue);

ssize_t
virtio_transport_seqpacket_dequeue(struct vsock_sock *vsk,
                                   struct msghdr *msg,
                                   int flags)
{
        if (flags & MSG_PEEK)
                return virtio_transport_seqpacket_do_peek(vsk, msg);
        else
                return virtio_transport_seqpacket_do_dequeue(vsk, msg, flags);
}
EXPORT_SYMBOL_GPL(virtio_transport_seqpacket_dequeue);

static u32 virtio_transport_tx_buf_size(struct virtio_vsock_sock *vvs)
{
        /* The peer advertises its receive buffer via peer_buf_alloc, but we
         * cap it to our local buf_alloc so a remote peer cannot force us to
         * queue more data than our own buffer configuration allows.
         */
        return min(vvs->peer_buf_alloc, vvs->buf_alloc);
}

int
virtio_transport_seqpacket_enqueue(struct vsock_sock *vsk,
                                   struct msghdr *msg,
                                   size_t len)
{
        struct virtio_vsock_sock *vvs = vsk->trans;

        spin_lock_bh(&vvs->tx_lock);

        if (len > virtio_transport_tx_buf_size(vvs)) {
                spin_unlock_bh(&vvs->tx_lock);
                return -EMSGSIZE;
        }

        spin_unlock_bh(&vvs->tx_lock);

        return virtio_transport_stream_enqueue(vsk, msg, len);
}
EXPORT_SYMBOL_GPL(virtio_transport_seqpacket_enqueue);

int
virtio_transport_dgram_dequeue(struct vsock_sock *vsk,
                               struct msghdr *msg,
                               size_t len, int flags)
{
        return -EOPNOTSUPP;
}
EXPORT_SYMBOL_GPL(virtio_transport_dgram_dequeue);

s64 virtio_transport_stream_has_data(struct vsock_sock *vsk)
{
        struct virtio_vsock_sock *vvs = vsk->trans;
        s64 bytes;

        spin_lock_bh(&vvs->rx_lock);
        bytes = vvs->rx_bytes;
        spin_unlock_bh(&vvs->rx_lock);

        return bytes;
}
EXPORT_SYMBOL_GPL(virtio_transport_stream_has_data);

u32 virtio_transport_seqpacket_has_data(struct vsock_sock *vsk)
{
        struct virtio_vsock_sock *vvs = vsk->trans;
        u32 msg_count;

        spin_lock_bh(&vvs->rx_lock);
        msg_count = vvs->msg_count;
        spin_unlock_bh(&vvs->rx_lock);

        return msg_count;
}
EXPORT_SYMBOL_GPL(virtio_transport_seqpacket_has_data);

static s64 virtio_transport_has_space(struct virtio_vsock_sock *vvs)
{
        s64 bytes;

        /* Use s64 arithmetic so if the peer shrinks peer_buf_alloc while
         * we have bytes in flight (tx_cnt - peer_fwd_cnt), the subtraction
         * does not underflow.
         */
        bytes = (s64)virtio_transport_tx_buf_size(vvs) -
                (vvs->tx_cnt - vvs->peer_fwd_cnt);
        if (bytes < 0)
                bytes = 0;

        return bytes;
}

s64 virtio_transport_stream_has_space(struct vsock_sock *vsk)
{
        struct virtio_vsock_sock *vvs = vsk->trans;
        s64 bytes;

        spin_lock_bh(&vvs->tx_lock);
        bytes = virtio_transport_has_space(vvs);
        spin_unlock_bh(&vvs->tx_lock);

        return bytes;
}
EXPORT_SYMBOL_GPL(virtio_transport_stream_has_space);

int virtio_transport_do_socket_init(struct vsock_sock *vsk,
                                    struct vsock_sock *psk)
{
        struct virtio_vsock_sock *vvs;

        vvs = kzalloc_obj(*vvs);
        if (!vvs)
                return -ENOMEM;

        vsk->trans = vvs;
        vvs->vsk = vsk;
        if (psk && psk->trans) {
                struct virtio_vsock_sock *ptrans = psk->trans;

                vvs->peer_buf_alloc = ptrans->peer_buf_alloc;
        }

        if (vsk->buffer_size > VIRTIO_VSOCK_MAX_BUF_SIZE)
                vsk->buffer_size = VIRTIO_VSOCK_MAX_BUF_SIZE;

        vvs->buf_alloc = vsk->buffer_size;

        spin_lock_init(&vvs->rx_lock);
        spin_lock_init(&vvs->tx_lock);
        skb_queue_head_init(&vvs->rx_queue);

        return 0;
}
EXPORT_SYMBOL_GPL(virtio_transport_do_socket_init);

/* sk_lock held by the caller */
void virtio_transport_notify_buffer_size(struct vsock_sock *vsk, u64 *val)
{
        struct virtio_vsock_sock *vvs = vsk->trans;

        if (*val > VIRTIO_VSOCK_MAX_BUF_SIZE)
                *val = VIRTIO_VSOCK_MAX_BUF_SIZE;

        vvs->buf_alloc = *val;

        virtio_transport_send_credit_update(vsk);
}
EXPORT_SYMBOL_GPL(virtio_transport_notify_buffer_size);

int
virtio_transport_notify_poll_in(struct vsock_sock *vsk,
                                size_t target,
                                bool *data_ready_now)
{
        *data_ready_now = vsock_stream_has_data(vsk) >= target;

        return 0;
}
EXPORT_SYMBOL_GPL(virtio_transport_notify_poll_in);

int
virtio_transport_notify_poll_out(struct vsock_sock *vsk,
                                 size_t target,
                                 bool *space_avail_now)
{
        s64 free_space;

        free_space = vsock_stream_has_space(vsk);
        if (free_space > 0)
                *space_avail_now = true;
        else if (free_space == 0)
                *space_avail_now = false;

        return 0;
}
EXPORT_SYMBOL_GPL(virtio_transport_notify_poll_out);

int virtio_transport_notify_recv_init(struct vsock_sock *vsk,
        size_t target, struct vsock_transport_recv_notify_data *data)
{
        return 0;
}
EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_init);

int virtio_transport_notify_recv_pre_block(struct vsock_sock *vsk,
        size_t target, struct vsock_transport_recv_notify_data *data)
{
        return 0;
}
EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_pre_block);

int virtio_transport_notify_recv_pre_dequeue(struct vsock_sock *vsk,
        size_t target, struct vsock_transport_recv_notify_data *data)
{
        return 0;
}
EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_pre_dequeue);

int virtio_transport_notify_recv_post_dequeue(struct vsock_sock *vsk,
        size_t target, ssize_t copied, bool data_read,
        struct vsock_transport_recv_notify_data *data)
{
        return 0;
}
EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_post_dequeue);

int virtio_transport_notify_send_init(struct vsock_sock *vsk,
        struct vsock_transport_send_notify_data *data)
{
        return 0;
}
EXPORT_SYMBOL_GPL(virtio_transport_notify_send_init);

int virtio_transport_notify_send_pre_block(struct vsock_sock *vsk,
        struct vsock_transport_send_notify_data *data)
{
        return 0;
}
EXPORT_SYMBOL_GPL(virtio_transport_notify_send_pre_block);

int virtio_transport_notify_send_pre_enqueue(struct vsock_sock *vsk,
        struct vsock_transport_send_notify_data *data)
{
        return 0;
}
EXPORT_SYMBOL_GPL(virtio_transport_notify_send_pre_enqueue);

int virtio_transport_notify_send_post_enqueue(struct vsock_sock *vsk,
        ssize_t written, struct vsock_transport_send_notify_data *data)
{
        return 0;
}
EXPORT_SYMBOL_GPL(virtio_transport_notify_send_post_enqueue);

u64 virtio_transport_stream_rcvhiwat(struct vsock_sock *vsk)
{
        return vsk->buffer_size;
}
EXPORT_SYMBOL_GPL(virtio_transport_stream_rcvhiwat);

bool virtio_transport_stream_is_active(struct vsock_sock *vsk)
{
        return true;
}
EXPORT_SYMBOL_GPL(virtio_transport_stream_is_active);

int virtio_transport_dgram_bind(struct vsock_sock *vsk,
                                struct sockaddr_vm *addr)
{
        return -EOPNOTSUPP;
}
EXPORT_SYMBOL_GPL(virtio_transport_dgram_bind);

bool virtio_transport_dgram_allow(struct vsock_sock *vsk, u32 cid, u32 port)
{
        return false;
}
EXPORT_SYMBOL_GPL(virtio_transport_dgram_allow);

int virtio_transport_connect(struct vsock_sock *vsk)
{
        struct virtio_vsock_pkt_info info = {
                .op = VIRTIO_VSOCK_OP_REQUEST,
                .vsk = vsk,
                .net = sock_net(sk_vsock(vsk)),
        };

        return virtio_transport_send_pkt_info(vsk, &info);
}
EXPORT_SYMBOL_GPL(virtio_transport_connect);

int virtio_transport_shutdown(struct vsock_sock *vsk, int mode)
{
        struct virtio_vsock_pkt_info info = {
                .op = VIRTIO_VSOCK_OP_SHUTDOWN,
                .flags = (mode & RCV_SHUTDOWN ?
                          VIRTIO_VSOCK_SHUTDOWN_RCV : 0) |
                         (mode & SEND_SHUTDOWN ?
                          VIRTIO_VSOCK_SHUTDOWN_SEND : 0),
                .vsk = vsk,
                .net = sock_net(sk_vsock(vsk)),
        };

        return virtio_transport_send_pkt_info(vsk, &info);
}
EXPORT_SYMBOL_GPL(virtio_transport_shutdown);

int
virtio_transport_dgram_enqueue(struct vsock_sock *vsk,
                               struct sockaddr_vm *remote_addr,
                               struct msghdr *msg,
                               size_t dgram_len)
{
        return -EOPNOTSUPP;
}
EXPORT_SYMBOL_GPL(virtio_transport_dgram_enqueue);

ssize_t
virtio_transport_stream_enqueue(struct vsock_sock *vsk,
                                struct msghdr *msg,
                                size_t len)
{
        struct virtio_vsock_pkt_info info = {
                .op = VIRTIO_VSOCK_OP_RW,
                .msg = msg,
                .pkt_len = len,
                .vsk = vsk,
                .net = sock_net(sk_vsock(vsk)),
        };

        return virtio_transport_send_pkt_info(vsk, &info);
}
EXPORT_SYMBOL_GPL(virtio_transport_stream_enqueue);

void virtio_transport_destruct(struct vsock_sock *vsk)
{
        struct virtio_vsock_sock *vvs = vsk->trans;

        virtio_transport_cancel_close_work(vsk, true);

        kfree(vvs);
        vsk->trans = NULL;
}
EXPORT_SYMBOL_GPL(virtio_transport_destruct);

ssize_t virtio_transport_unsent_bytes(struct vsock_sock *vsk)
{
        struct virtio_vsock_sock *vvs = vsk->trans;
        size_t ret;

        spin_lock_bh(&vvs->tx_lock);
        ret = vvs->bytes_unsent;
        spin_unlock_bh(&vvs->tx_lock);

        return ret;
}
EXPORT_SYMBOL_GPL(virtio_transport_unsent_bytes);

static int virtio_transport_reset(struct vsock_sock *vsk,
                                  struct sk_buff *skb)
{
        struct virtio_vsock_pkt_info info = {
                .op = VIRTIO_VSOCK_OP_RST,
                .reply = !!skb,
                .vsk = vsk,
                .net = sock_net(sk_vsock(vsk)),
        };

        /* Send RST only if the original pkt is not a RST pkt */
        if (skb && le16_to_cpu(virtio_vsock_hdr(skb)->op) == VIRTIO_VSOCK_OP_RST)
                return 0;

        return virtio_transport_send_pkt_info(vsk, &info);
}

/* Normally packets are associated with a socket.  There may be no socket if an
 * attempt was made to connect to a socket that does not exist.
 *
 * net refers to the namespace of whoever sent the invalid message. For
 * loopback, this is the namespace of the socket. For vhost, this is the
 * namespace of the VM (i.e., vhost_vsock).
 */
static int virtio_transport_reset_no_sock(const struct virtio_transport *t,
                                          struct sk_buff *skb, struct net *net)
{
        struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
        struct virtio_vsock_pkt_info info = {
                .op = VIRTIO_VSOCK_OP_RST,
                .type = le16_to_cpu(hdr->type),
                .reply = true,

                /* Set sk owner to socket we are replying to (may be NULL for
                 * non-loopback). This keeps a reference to the sock and
                 * sock_net(sk) until the reply skb is freed.
                 */
                .vsk = vsock_sk(skb->sk),

                /* net is not defined here because we pass it directly to
                 * t->send_pkt(), instead of relying on
                 * virtio_transport_send_pkt_info() to pass it. It is not needed
                 * by virtio_transport_alloc_skb().
                 */
        };
        struct sk_buff *reply;

        /* Send RST only if the original pkt is not a RST pkt */
        if (le16_to_cpu(hdr->op) == VIRTIO_VSOCK_OP_RST)
                return 0;

        if (!t)
                return -ENOTCONN;

        reply = virtio_transport_alloc_skb(&info, 0, false, NULL,
                                           le64_to_cpu(hdr->dst_cid),
                                           le32_to_cpu(hdr->dst_port),
                                           le64_to_cpu(hdr->src_cid),
                                           le32_to_cpu(hdr->src_port));
        if (!reply)
                return -ENOMEM;

        return t->send_pkt(reply, net);
}

/* This function should be called with sk_lock held and SOCK_DONE set */
static void virtio_transport_remove_sock(struct vsock_sock *vsk)
{
        struct virtio_vsock_sock *vvs = vsk->trans;

        /* We don't need to take rx_lock, as the socket is closing and we are
         * removing it.
         */
        __skb_queue_purge(&vvs->rx_queue);
        vsock_remove_sock(vsk);
}

static void virtio_transport_cancel_close_work(struct vsock_sock *vsk,
                                               bool cancel_timeout)
{
        struct sock *sk = sk_vsock(vsk);

        if (vsk->close_work_scheduled &&
            (!cancel_timeout || cancel_delayed_work(&vsk->close_work))) {
                vsk->close_work_scheduled = false;

                virtio_transport_remove_sock(vsk);

                /* Release refcnt obtained when we scheduled the timeout */
                sock_put(sk);
        }
}

static void virtio_transport_do_close(struct vsock_sock *vsk,
                                      bool cancel_timeout)
{
        struct sock *sk = sk_vsock(vsk);

        sock_set_flag(sk, SOCK_DONE);
        WRITE_ONCE(vsk->peer_shutdown, SHUTDOWN_MASK);
        if (vsock_stream_has_data(vsk) <= 0)
                sk->sk_state = TCP_CLOSING;
        sk->sk_state_change(sk);

        virtio_transport_cancel_close_work(vsk, cancel_timeout);
}

static void virtio_transport_close_timeout(struct work_struct *work)
{
        struct vsock_sock *vsk =
                container_of(work, struct vsock_sock, close_work.work);
        struct sock *sk = sk_vsock(vsk);

        sock_hold(sk);
        lock_sock(sk);

        if (!sock_flag(sk, SOCK_DONE)) {
                (void)virtio_transport_reset(vsk, NULL);

                virtio_transport_do_close(vsk, false);
        }

        vsk->close_work_scheduled = false;

        release_sock(sk);
        sock_put(sk);
}

/* User context, vsk->sk is locked */
static bool virtio_transport_close(struct vsock_sock *vsk)
{
        struct sock *sk = &vsk->sk;

        if (!(sk->sk_state == TCP_ESTABLISHED ||
              sk->sk_state == TCP_CLOSING))
                return true;

        /* Already received SHUTDOWN from peer, reply with RST */
        if ((vsk->peer_shutdown & SHUTDOWN_MASK) == SHUTDOWN_MASK) {
                (void)virtio_transport_reset(vsk, NULL);
                return true;
        }

        if ((sk->sk_shutdown & SHUTDOWN_MASK) != SHUTDOWN_MASK)
                (void)virtio_transport_shutdown(vsk, SHUTDOWN_MASK);

        if (!(current->flags & PF_EXITING))
                vsock_linger(sk);

        if (sock_flag(sk, SOCK_DONE)) {
                return true;
        }

        sock_hold(sk);
        INIT_DELAYED_WORK(&vsk->close_work,
                          virtio_transport_close_timeout);
        vsk->close_work_scheduled = true;
        schedule_delayed_work(&vsk->close_work, VSOCK_CLOSE_TIMEOUT);
        return false;
}

void virtio_transport_release(struct vsock_sock *vsk)
{
        struct sock *sk = &vsk->sk;
        bool remove_sock = true;

        if (sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET)
                remove_sock = virtio_transport_close(vsk);

        if (remove_sock) {
                sock_set_flag(sk, SOCK_DONE);
                virtio_transport_remove_sock(vsk);
        }
}
EXPORT_SYMBOL_GPL(virtio_transport_release);

static int
virtio_transport_recv_connecting(struct sock *sk,
                                 struct sk_buff *skb)
{
        struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
        struct vsock_sock *vsk = vsock_sk(sk);
        int skerr;
        int err;

        switch (le16_to_cpu(hdr->op)) {
        case VIRTIO_VSOCK_OP_RESPONSE:
                sk->sk_state = TCP_ESTABLISHED;
                sk->sk_socket->state = SS_CONNECTED;
                vsock_insert_connected(vsk);
                sk->sk_state_change(sk);
                break;
        case VIRTIO_VSOCK_OP_INVALID:
                break;
        case VIRTIO_VSOCK_OP_RST:
                skerr = ECONNRESET;
                err = 0;
                goto destroy;
        default:
                skerr = EPROTO;
                err = -EINVAL;
                goto destroy;
        }
        return 0;

destroy:
        virtio_transport_reset(vsk, skb);
        sk->sk_state = TCP_CLOSE;
        sk->sk_err = skerr;
        sk_error_report(sk);
        return err;
}

static bool
virtio_transport_recv_enqueue(struct vsock_sock *vsk,
                              struct sk_buff *skb)
{
        struct virtio_vsock_sock *vvs = vsk->trans;
        bool can_enqueue, free_pkt = false;
        u32 len, queue_max, queue_len;
        struct virtio_vsock_hdr *hdr;

        hdr = virtio_vsock_hdr(skb);
        len = le32_to_cpu(hdr->len);

        /* virtio_transport_inc_rx_pkt() rejects packets when the per-skb
         * overhead (skb_queue_len * SKB_TRUESIZE(0)) exceeds buf_alloc.
         * Proactively collapse the queue before that happens.
         * No rx_lock needed: lock_sock is held by caller, preventing
         * concurrent enqueue or dequeue.
         */
        queue_max = vvs->buf_alloc / SKB_TRUESIZE(0);
        queue_len = skb_queue_len(&vvs->rx_queue);
        if (queue_len >= queue_max) {
                /* Walking a large queue may take a significant amount of time
                 * and cache misses, causing traffic burstiness. Limit the
                 * collapse to freeing room for this packet and the next one.
                 * It may free more to fill each collapsed skb to capacity.
                 */
                virtio_transport_collapse_rx_queue(vvs, queue_len + 2 - queue_max);
        }

        spin_lock_bh(&vvs->rx_lock);

        can_enqueue = virtio_transport_inc_rx_pkt(vvs, len);
        if (!can_enqueue)
                goto out;

        if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM)
                vvs->msg_count++;

        /* Try to copy small packets into the buffer of last packet queued,
         * to avoid wasting memory queueing the entire buffer with a small
         * payload. Skip non-linear (e.g. zerocopy) skbs; these carry payload
         * in skb_shinfo.
         */
        if (len <= GOOD_COPY_LEN && !skb_queue_empty(&vvs->rx_queue) &&
            !skb_is_nonlinear(skb)) {
                struct virtio_vsock_hdr *last_hdr;
                struct sk_buff *last_skb;

                last_skb = skb_peek_tail(&vvs->rx_queue);
                last_hdr = virtio_vsock_hdr(last_skb);

                /* If there is space in the last packet queued, we copy the
                 * new packet in its buffer. We avoid this if the last packet
                 * queued has VIRTIO_VSOCK_SEQ_EOM set, because this is
                 * delimiter of SEQPACKET message, so 'pkt' is the first packet
                 * of a new message.
                 */
                if (skb->len < skb_tailroom(last_skb) &&
                    !(le32_to_cpu(last_hdr->flags) & VIRTIO_VSOCK_SEQ_EOM)) {
                        memcpy(skb_put(last_skb, skb->len), skb->data, skb->len);
                        free_pkt = true;
                        last_hdr->flags |= hdr->flags;
                        le32_add_cpu(&last_hdr->len, len);
                        goto out;
                }
        }

        __skb_queue_tail(&vvs->rx_queue, skb);

out:
        spin_unlock_bh(&vvs->rx_lock);
        if (free_pkt)
                kfree_skb(skb);

        return can_enqueue;
}

static int
virtio_transport_recv_connected(struct sock *sk,
                                struct sk_buff *skb)
{
        struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
        struct vsock_sock *vsk = vsock_sk(sk);
        int err = 0;

        switch (le16_to_cpu(hdr->op)) {
        case VIRTIO_VSOCK_OP_RW:
                if (!virtio_transport_recv_enqueue(vsk, skb)) {
                        /* There is no more space to queue the packet, so let's
                         * close the connection; otherwise, we'll lose data.
                         */
                        (void)virtio_transport_reset(vsk, skb);
                        virtio_transport_do_close(vsk, true);
                        sk->sk_err = ENOBUFS;
                        sk_error_report(sk);
                        vsock_remove_sock(vsk);
                        break;
                }
                vsock_data_ready(sk);
                return err;
        case VIRTIO_VSOCK_OP_CREDIT_REQUEST:
                virtio_transport_send_credit_update(vsk);
                break;
        case VIRTIO_VSOCK_OP_CREDIT_UPDATE:
                sk->sk_write_space(sk);
                break;
        case VIRTIO_VSOCK_OP_SHUTDOWN: {
                u32 peer_shutdown = READ_ONCE(vsk->peer_shutdown);

                if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SHUTDOWN_RCV)
                        peer_shutdown |= RCV_SHUTDOWN;
                if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SHUTDOWN_SEND)
                        peer_shutdown |= SEND_SHUTDOWN;
                WRITE_ONCE(vsk->peer_shutdown, peer_shutdown);
                if (peer_shutdown == SHUTDOWN_MASK) {
                        if (vsock_stream_has_data(vsk) <= 0 && !sock_flag(sk, SOCK_DONE)) {
                                (void)virtio_transport_reset(vsk, NULL);
                                virtio_transport_do_close(vsk, true);
                        }
                        /* Remove this socket anyway because the remote peer sent
                         * the shutdown. This way a new connection will succeed
                         * if the remote peer uses the same source port,
                         * even if the old socket is still unreleased, but now disconnected.
                         */
                        vsock_remove_sock(vsk);
                }
                if (le32_to_cpu(virtio_vsock_hdr(skb)->flags))
                        sk->sk_state_change(sk);
                break;
        }
        case VIRTIO_VSOCK_OP_RST:
                virtio_transport_do_close(vsk, true);
                break;
        default:
                err = -EINVAL;
                break;
        }

        kfree_skb(skb);
        return err;
}

static void
virtio_transport_recv_disconnecting(struct sock *sk,
                                    struct sk_buff *skb)
{
        struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
        struct vsock_sock *vsk = vsock_sk(sk);

        if (le16_to_cpu(hdr->op) == VIRTIO_VSOCK_OP_RST)
                virtio_transport_do_close(vsk, true);
}

static int
virtio_transport_send_response(struct vsock_sock *vsk,
                               struct sk_buff *skb)
{
        struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
        struct virtio_vsock_pkt_info info = {
                .op = VIRTIO_VSOCK_OP_RESPONSE,
                .remote_cid = le64_to_cpu(hdr->src_cid),
                .remote_port = le32_to_cpu(hdr->src_port),
                .reply = true,
                .vsk = vsk,
                .net = sock_net(sk_vsock(vsk)),
        };

        return virtio_transport_send_pkt_info(vsk, &info);
}

static bool virtio_transport_space_update(struct sock *sk,
                                          struct sk_buff *skb)
{
        struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
        struct vsock_sock *vsk = vsock_sk(sk);
        struct virtio_vsock_sock *vvs = vsk->trans;
        bool space_available;

        /* Listener sockets are not associated with any transport, so we are
         * not able to take the state to see if there is space available in the
         * remote peer, but since they are only used to receive requests, we
         * can assume that there is always space available in the other peer.
         */
        if (!vvs)
                return true;

        /* buf_alloc and fwd_cnt is always included in the hdr */
        spin_lock_bh(&vvs->tx_lock);
        vvs->peer_buf_alloc = le32_to_cpu(hdr->buf_alloc);
        vvs->peer_fwd_cnt = le32_to_cpu(hdr->fwd_cnt);
        space_available = virtio_transport_has_space(vvs);
        spin_unlock_bh(&vvs->tx_lock);
        return space_available;
}

/* Handle server socket */
static int
virtio_transport_recv_listen(struct sock *sk, struct sk_buff *skb,
                             struct virtio_transport *t)
{
        struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
        struct vsock_sock *vsk = vsock_sk(sk);
        struct vsock_sock *vchild;
        struct sock *child;
        int ret;

        if (le16_to_cpu(hdr->op) != VIRTIO_VSOCK_OP_REQUEST) {
                virtio_transport_reset_no_sock(t, skb, sock_net(sk));
                return -EINVAL;
        }

        if (sk_acceptq_is_full(sk)) {
                virtio_transport_reset_no_sock(t, skb, sock_net(sk));
                return -ENOMEM;
        }

        /* __vsock_release() might have already flushed accept_queue.
         * Subsequent enqueues would lead to a memory leak.
         */
        if (sk->sk_shutdown == SHUTDOWN_MASK) {
                virtio_transport_reset_no_sock(t, skb, sock_net(sk));
                return -ESHUTDOWN;
        }

        child = vsock_create_connected(sk);
        if (!child) {
                virtio_transport_reset_no_sock(t, skb, sock_net(sk));
                return -ENOMEM;
        }

        lock_sock_nested(child, SINGLE_DEPTH_NESTING);

        child->sk_state = TCP_ESTABLISHED;

        vchild = vsock_sk(child);
        vsock_addr_init(&vchild->local_addr, le64_to_cpu(hdr->dst_cid),
                        le32_to_cpu(hdr->dst_port));
        vsock_addr_init(&vchild->remote_addr, le64_to_cpu(hdr->src_cid),
                        le32_to_cpu(hdr->src_port));

        ret = vsock_assign_transport(vchild, vsk);
        /* Transport assigned (looking at remote_addr) must be the same
         * where we received the request.
         */
        if (ret || vchild->transport != &t->transport) {
                release_sock(child);
                virtio_transport_reset_no_sock(t, skb, sock_net(sk));
                sock_put(child);
                return ret;
        }

        if (virtio_transport_space_update(child, skb))
                child->sk_write_space(child);

        vsock_insert_connected(vchild);
        vsock_enqueue_accept(sk, child);
        virtio_transport_send_response(vchild, skb);

        release_sock(child);

        sk->sk_data_ready(sk);
        return 0;
}

static bool virtio_transport_valid_type(u16 type)
{
        return (type == VIRTIO_VSOCK_TYPE_STREAM) ||
               (type == VIRTIO_VSOCK_TYPE_SEQPACKET);
}

/* We are under the virtio-vsock's vsock->rx_lock or vhost-vsock's vq->mutex
 * lock.
 */
void virtio_transport_recv_pkt(struct virtio_transport *t,
                               struct sk_buff *skb, struct net *net)
{
        struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
        struct sockaddr_vm src, dst;
        struct vsock_sock *vsk;
        struct sock *sk;
        bool space_available;

        vsock_addr_init(&src, le64_to_cpu(hdr->src_cid),
                        le32_to_cpu(hdr->src_port));
        vsock_addr_init(&dst, le64_to_cpu(hdr->dst_cid),
                        le32_to_cpu(hdr->dst_port));

        trace_virtio_transport_recv_pkt(src.svm_cid, src.svm_port,
                                        dst.svm_cid, dst.svm_port,
                                        le32_to_cpu(hdr->len),
                                        le16_to_cpu(hdr->type),
                                        le16_to_cpu(hdr->op),
                                        le32_to_cpu(hdr->flags),
                                        le32_to_cpu(hdr->buf_alloc),
                                        le32_to_cpu(hdr->fwd_cnt));

        if (!virtio_transport_valid_type(le16_to_cpu(hdr->type))) {
                (void)virtio_transport_reset_no_sock(t, skb, net);
                goto free_pkt;
        }

        /* The socket must be in connected or bound table
         * otherwise send reset back
         */
        sk = vsock_find_connected_socket_net(&src, &dst, net);
        if (!sk) {
                sk = vsock_find_bound_socket_net(&dst, net);
                if (!sk) {
                        (void)virtio_transport_reset_no_sock(t, skb, net);
                        goto free_pkt;
                }
        }

        if (virtio_transport_get_type(sk) != le16_to_cpu(hdr->type)) {
                (void)virtio_transport_reset_no_sock(t, skb, net);
                sock_put(sk);
                goto free_pkt;
        }

        if (!skb_set_owner_sk_safe(skb, sk)) {
                WARN_ONCE(1, "receiving vsock socket has sk_refcnt == 0\n");
                goto free_pkt;
        }

        vsk = vsock_sk(sk);

        lock_sock(sk);

        /* Check if sk has been closed or assigned to another transport before
         * lock_sock (note: listener sockets are not assigned to any transport)
         */
        if (sock_flag(sk, SOCK_DONE) ||
            (sk->sk_state != TCP_LISTEN &&
             !vsock_check_source(vsk, &t->transport, &src))) {
                (void)virtio_transport_reset_no_sock(t, skb, net);
                release_sock(sk);
                sock_put(sk);
                goto free_pkt;
        }

        space_available = virtio_transport_space_update(sk, skb);

        /* Update CID in case it has changed after a transport reset event */
        if (vsk->local_addr.svm_cid != VMADDR_CID_ANY)
                vsk->local_addr.svm_cid = dst.svm_cid;

        if (space_available)
                sk->sk_write_space(sk);

        switch (sk->sk_state) {
        case TCP_LISTEN:
                virtio_transport_recv_listen(sk, skb, t);
                kfree_skb(skb);
                break;
        case TCP_SYN_SENT:
                virtio_transport_recv_connecting(sk, skb);
                kfree_skb(skb);
                break;
        case TCP_ESTABLISHED:
                virtio_transport_recv_connected(sk, skb);
                break;
        case TCP_CLOSING:
                virtio_transport_recv_disconnecting(sk, skb);
                kfree_skb(skb);
                break;
        default:
                (void)virtio_transport_reset_no_sock(t, skb, net);
                kfree_skb(skb);
                break;
        }

        release_sock(sk);

        /* Release refcnt obtained when we fetched this socket out of the
         * bound or connected list.
         */
        sock_put(sk);
        return;

free_pkt:
        kfree_skb(skb);
}
EXPORT_SYMBOL_GPL(virtio_transport_recv_pkt);

/* Remove skbs found in a queue that have a vsk that matches.
 *
 * Each skb is freed.
 *
 * Returns the count of skbs that were reply packets.
 */
int virtio_transport_purge_skbs(void *vsk, struct sk_buff_head *queue)
{
        struct sk_buff_head freeme;
        struct sk_buff *skb, *tmp;
        int cnt = 0;

        skb_queue_head_init(&freeme);

        spin_lock_bh(&queue->lock);
        skb_queue_walk_safe(queue, skb, tmp) {
                if (vsock_sk(skb->sk) != vsk)
                        continue;

                __skb_unlink(skb, queue);
                __skb_queue_tail(&freeme, skb);

                if (virtio_vsock_skb_reply(skb))
                        cnt++;
        }
        spin_unlock_bh(&queue->lock);

        __skb_queue_purge(&freeme);

        return cnt;
}
EXPORT_SYMBOL_GPL(virtio_transport_purge_skbs);

int virtio_transport_read_skb(struct vsock_sock *vsk, skb_read_actor_t recv_actor)
{
        struct virtio_vsock_sock *vvs = vsk->trans;
        struct sock *sk = sk_vsock(vsk);
        struct virtio_vsock_hdr *hdr;
        struct sk_buff *skb;
        u32 pkt_len;
        int off = 0;
        int err;

        spin_lock_bh(&vvs->rx_lock);
        /* Use __skb_recv_datagram() for race-free handling of the receive. It
         * works for types other than dgrams.
         */
        skb = __skb_recv_datagram(sk, &vvs->rx_queue, MSG_DONTWAIT, &off, &err);
        if (!skb) {
                spin_unlock_bh(&vvs->rx_lock);
                return err;
        }

        hdr = virtio_vsock_hdr(skb);
        if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM)
                vvs->msg_count--;

        pkt_len = le32_to_cpu(hdr->len);
        virtio_transport_dec_rx_pkt(vvs, pkt_len, pkt_len);
        spin_unlock_bh(&vvs->rx_lock);

        virtio_transport_send_credit_update(vsk);

        return recv_actor(sk, skb);
}
EXPORT_SYMBOL_GPL(virtio_transport_read_skb);

int virtio_transport_notify_set_rcvlowat(struct vsock_sock *vsk, int val)
{
        struct virtio_vsock_sock *vvs = vsk->trans;
        bool send_update;

        spin_lock_bh(&vvs->rx_lock);

        /* If number of available bytes is less than new SO_RCVLOWAT value,
         * kick sender to send more data, because sender may sleep in its
         * 'send()' syscall waiting for enough space at our side. Also
         * don't send credit update when peer already knows actual value -
         * such transmission will be useless.
         */
        send_update = (vvs->rx_bytes < val) &&
                      (vvs->fwd_cnt != vvs->last_fwd_cnt);

        spin_unlock_bh(&vvs->rx_lock);

        if (send_update) {
                int err;

                err = virtio_transport_send_credit_update(vsk);
                if (err < 0)
                        return err;
        }

        return 0;
}
EXPORT_SYMBOL_GPL(virtio_transport_notify_set_rcvlowat);

MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Asias He");
MODULE_DESCRIPTION("common code for virtio vsock");
























































   20 








   17 
   17 




    8 

























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
// SPDX-License-Identifier: GPL-2.0-only
/* iptables module to match on related connections */
/*
 * (C) 2001 Martin Josefsson <gandalf@wlug.westbo.se>
 */
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/netfilter.h>
#include <net/netfilter/nf_conntrack.h>
#include <net/netfilter/nf_conntrack_core.h>
#include <net/netfilter/nf_conntrack_helper.h>
#include <linux/netfilter/x_tables.h>
#include <linux/netfilter/xt_helper.h>

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Martin Josefsson <gandalf@netfilter.org>");
MODULE_DESCRIPTION("Xtables: Related connection matching");
MODULE_ALIAS("ipt_helper");
MODULE_ALIAS("ip6t_helper");


static bool
helper_mt(const struct sk_buff *skb, struct xt_action_param *par)
{
        const struct xt_helper_info *info = par->matchinfo;
        const struct nf_conn *ct;
        const struct nf_conn_help *master_help;
        const struct nf_conntrack_helper *helper;
        enum ip_conntrack_info ctinfo;
        bool ret = info->invert;

        ct = nf_ct_get(skb, &ctinfo);
        if (!ct || !ct->master)
                return ret;

        master_help = nfct_help(ct->master);
        if (!master_help)
                return ret;

        /* rcu_read_lock()ed by nf_hook_thresh */
        helper = rcu_dereference(master_help->helper);
        if (!helper)
                return ret;

        if (info->name[0] == '\0')
                ret = !ret;
        else
                ret ^= !strncmp(helper->name, info->name,
                                strlen(helper->name));
        return ret;
}

static int helper_mt_check(const struct xt_mtchk_param *par)
{
        struct xt_helper_info *info = par->matchinfo;
        int ret;

        ret = nf_ct_netns_get(par->net, par->family);
        if (ret < 0) {
                pr_info_ratelimited("cannot load conntrack support for proto=%u\n",
                                    par->family);
                return ret;
        }
        info->name[sizeof(info->name) - 1] = '\0';
        return 0;
}

static void helper_mt_destroy(const struct xt_mtdtor_param *par)
{
        nf_ct_netns_put(par->net, par->family);
}

static struct xt_match helper_mt_reg __read_mostly = {
        .name       = "helper",
        .revision   = 0,
        .family     = NFPROTO_UNSPEC,
        .checkentry = helper_mt_check,
        .match      = helper_mt,
        .destroy    = helper_mt_destroy,
        .matchsize  = sizeof(struct xt_helper_info),
        .me         = THIS_MODULE,
};

static int __init helper_mt_init(void)
{
        return xt_register_match(&helper_mt_reg);
}

static void __exit helper_mt_exit(void)
{
        xt_unregister_match(&helper_mt_reg);
}

module_init(helper_mt_init);
module_exit(helper_mt_exit);





























    5 






    5 




    5 
    4 

    5 












    5 










    1 


    1 
    1 


    1 





    1 






















































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
// SPDX-License-Identifier: GPL-2.0-or-later
/* RxRPC packet reception
 *
 * Copyright (C) 2007, 2016, 2022 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include "ar-internal.h"

static int rxrpc_input_packet_on_conn(struct rxrpc_connection *conn,
                                      struct sockaddr_rxrpc *peer_srx,
                                      struct sk_buff *skb);

/*
 * handle data received on the local endpoint
 * - may be called in interrupt context
 *
 * [!] Note that as this is called from the encap_rcv hook, the socket is not
 * held locked by the caller and nothing prevents sk_user_data on the UDP from
 * being cleared in the middle of processing this function.
 *
 * Called with the RCU read lock held from the IP layer via UDP.
 */
int rxrpc_encap_rcv(struct sock *udp_sk, struct sk_buff *skb)
{
        struct sk_buff_head *rx_queue;
        struct rxrpc_local *local = rcu_dereference_sk_user_data(udp_sk);
        struct task_struct *io_thread;

        if (unlikely(!local)) {
                kfree_skb(skb);
                return 0;
        }
        io_thread = READ_ONCE(local->io_thread);
        if (!io_thread) {
                kfree_skb(skb);
                return 0;
        }
        if (skb->tstamp == 0)
                skb->tstamp = ktime_get_real();

        skb->mark = RXRPC_SKB_MARK_PACKET;
        rxrpc_new_skb(skb, rxrpc_skb_new_encap_rcv);
        rx_queue = &local->rx_queue;
#ifdef CONFIG_AF_RXRPC_INJECT_RX_DELAY
        if (rxrpc_inject_rx_delay ||
            !skb_queue_empty(&local->rx_delay_queue)) {
                skb->tstamp = ktime_add_ms(skb->tstamp, rxrpc_inject_rx_delay);
                rx_queue = &local->rx_delay_queue;
        }
#endif

        skb_queue_tail(rx_queue, skb);
        wake_up_process(io_thread);
        return 0;
}

/*
 * Handle an error received on the local endpoint.
 */
void rxrpc_error_report(struct sock *sk)
{
        struct rxrpc_local *local;
        struct sk_buff *skb;

        rcu_read_lock();
        local = rcu_dereference_sk_user_data(sk);
        if (unlikely(!local)) {
                rcu_read_unlock();
                return;
        }

        while ((skb = skb_dequeue(&sk->sk_error_queue))) {
                skb->mark = RXRPC_SKB_MARK_ERROR;
                rxrpc_new_skb(skb, rxrpc_skb_new_error_report);
                skb_queue_tail(&local->rx_queue, skb);
        }

        rxrpc_wake_up_io_thread(local);
        rcu_read_unlock();
}

/*
 * Directly produce an abort from a packet.
 */
bool rxrpc_direct_abort(struct sk_buff *skb, enum rxrpc_abort_reason why,
                        s32 abort_code, int err)
{
        struct rxrpc_skb_priv *sp = rxrpc_skb(skb);

        trace_rxrpc_abort(0, why, sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq,
                          abort_code, err);
        skb->mark = RXRPC_SKB_MARK_REJECT_ABORT;
        skb->priority = abort_code;
        return false;
}

/*
 * Directly produce a connection abort from a packet.
 */
bool rxrpc_direct_conn_abort(struct sk_buff *skb, enum rxrpc_abort_reason why,
                             s32 abort_code, int err)
{
        struct rxrpc_skb_priv *sp = rxrpc_skb(skb);

        trace_rxrpc_abort(0, why, sp->hdr.cid, 0, sp->hdr.seq, abort_code, err);
        skb->mark = RXRPC_SKB_MARK_REJECT_CONN_ABORT;
        skb->priority = abort_code;
        return false;
}

static bool rxrpc_bad_message(struct sk_buff *skb, enum rxrpc_abort_reason why)
{
        return rxrpc_direct_abort(skb, why, RX_PROTOCOL_ERROR, -EBADMSG);
}

#define just_discard true

/*
 * Process event packets targeted at a local endpoint.
 */
static bool rxrpc_input_version(struct rxrpc_local *local, struct sk_buff *skb)
{
        struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
        char v;

        _enter("");

        rxrpc_see_skb(skb, rxrpc_skb_see_version);
        if (skb_copy_bits(skb, sizeof(struct rxrpc_wire_header), &v, 1) >= 0) {
                if (v == 0)
                        rxrpc_send_version_request(local, &sp->hdr, skb);
        }

        return true;
}

/*
 * Extract the wire header from a packet and translate the byte order.
 */
static bool rxrpc_extract_header(struct rxrpc_skb_priv *sp,
                                 struct sk_buff *skb)
{
        struct rxrpc_wire_header whdr;
        struct rxrpc_ackpacket ack;

        /* dig out the RxRPC connection details */
        if (skb_copy_bits(skb, 0, &whdr, sizeof(whdr)) < 0)
                return rxrpc_bad_message(skb, rxrpc_badmsg_short_hdr);

        memset(sp, 0, sizeof(*sp));
        sp->hdr.epoch                = ntohl(whdr.epoch);
        sp->hdr.cid                = ntohl(whdr.cid);
        sp->hdr.callNumber        = ntohl(whdr.callNumber);
        sp->hdr.seq                = ntohl(whdr.seq);
        sp->hdr.serial                = ntohl(whdr.serial);
        sp->hdr.flags                = whdr.flags;
        sp->hdr.type                = whdr.type;
        sp->hdr.userStatus        = whdr.userStatus;
        sp->hdr.securityIndex        = whdr.securityIndex;
        sp->hdr._rsvd                = ntohs(whdr._rsvd);
        sp->hdr.serviceId        = ntohs(whdr.serviceId);

        if (sp->hdr.type == RXRPC_PACKET_TYPE_ACK) {
                if (skb_copy_bits(skb, sizeof(whdr), &ack, sizeof(ack)) < 0)
                        return rxrpc_bad_message(skb, rxrpc_badmsg_short_ack);
                sp->ack.first_ack        = ntohl(ack.firstPacket);
                sp->ack.prev_ack        = ntohl(ack.previousPacket);
                sp->ack.acked_serial        = ntohl(ack.serial);
                sp->ack.reason                = ack.reason;
                sp->ack.nr_acks                = ack.nAcks;
        }
        return true;
}

/*
 * Extract the abort code from an ABORT packet and stash it in skb->priority.
 */
static bool rxrpc_extract_abort(struct sk_buff *skb)
{
        __be32 wtmp;

        if (skb_copy_bits(skb, sizeof(struct rxrpc_wire_header),
                          &wtmp, sizeof(wtmp)) < 0)
                return false;
        skb->priority = ntohl(wtmp);
        return true;
}

/*
 * Process packets received on the local endpoint
 */
static bool rxrpc_input_packet(struct rxrpc_local *local, struct sk_buff *skb)
{
        struct rxrpc_connection *conn;
        struct sockaddr_rxrpc peer_srx;
        struct rxrpc_skb_priv *sp;
        struct rxrpc_peer *peer = NULL;
        bool ret = false;

        skb_pull(skb, sizeof(struct udphdr));

        sp = rxrpc_skb(skb);

        /* dig out the RxRPC connection details */
        if (!rxrpc_extract_header(sp, skb))
                return just_discard;

        if (IS_ENABLED(CONFIG_AF_RXRPC_INJECT_LOSS)) {
                static int lose;
                if ((lose++ & 7) == 7) {
                        trace_rxrpc_rx_lose(sp);
                        return just_discard;
                }
        }

        trace_rxrpc_rx_packet(sp);

        switch (sp->hdr.type) {
        case RXRPC_PACKET_TYPE_VERSION:
                if (rxrpc_to_client(sp))
                        return just_discard;
                return rxrpc_input_version(local, skb);

        case RXRPC_PACKET_TYPE_BUSY:
                if (rxrpc_to_server(sp))
                        return just_discard;
                fallthrough;
        case RXRPC_PACKET_TYPE_ACK:
        case RXRPC_PACKET_TYPE_ACKALL:
                if (sp->hdr.callNumber == 0)
                        return rxrpc_bad_message(skb, rxrpc_badmsg_zero_call);
                break;
        case RXRPC_PACKET_TYPE_ABORT:
                if (!rxrpc_extract_abort(skb))
                        return just_discard; /* Just discard if malformed */
                break;

        case RXRPC_PACKET_TYPE_DATA:
                if (sp->hdr.callNumber == 0)
                        return rxrpc_bad_message(skb, rxrpc_badmsg_zero_call);
                if (sp->hdr.seq == 0)
                        return rxrpc_bad_message(skb, rxrpc_badmsg_zero_seq);
                break;

        case RXRPC_PACKET_TYPE_CHALLENGE:
                if (rxrpc_to_server(sp))
                        return just_discard;
                break;
        case RXRPC_PACKET_TYPE_RESPONSE:
                if (rxrpc_to_client(sp))
                        return just_discard;
                break;

                /* Packet types 9-11 should just be ignored. */
        case RXRPC_PACKET_TYPE_PARAMS:
        case RXRPC_PACKET_TYPE_10:
        case RXRPC_PACKET_TYPE_11:
                return just_discard;

        default:
                return rxrpc_bad_message(skb, rxrpc_badmsg_unsupported_packet);
        }

        if (sp->hdr.serviceId == 0)
                return rxrpc_bad_message(skb, rxrpc_badmsg_zero_service);

        if (WARN_ON_ONCE(rxrpc_extract_addr_from_skb(&peer_srx, skb) < 0))
                return just_discard; /* Unsupported address type. */

        if (peer_srx.transport.family != local->srx.transport.family &&
            (peer_srx.transport.family == AF_INET &&
             local->srx.transport.family != AF_INET6)) {
                pr_warn_ratelimited("AF_RXRPC: Protocol mismatch %u not %u\n",
                                    peer_srx.transport.family,
                                    local->srx.transport.family);
                return just_discard; /* Wrong address type. */
        }

        if (rxrpc_to_client(sp)) {
                rcu_read_lock();
                conn = rxrpc_find_client_connection_rcu(local, &peer_srx, skb);
                conn = rxrpc_get_connection_maybe(conn, rxrpc_conn_get_call_input);
                rcu_read_unlock();
                if (!conn)
                        return rxrpc_protocol_error(skb, rxrpc_eproto_no_client_conn);

                ret = rxrpc_input_packet_on_conn(conn, &peer_srx, skb);
                rxrpc_put_connection(conn, rxrpc_conn_put_call_input);
                return ret;
        }

        /* We need to look up service connections by the full protocol
         * parameter set.  We look up the peer first as an intermediate step
         * and then the connection from the peer's tree.
         */
        rcu_read_lock();

        peer = rxrpc_lookup_peer_rcu(local, &peer_srx);
        if (!peer) {
                rcu_read_unlock();
                return rxrpc_new_incoming_call(local, NULL, NULL, &peer_srx, skb);
        }

        conn = rxrpc_find_service_conn_rcu(peer, skb);
        conn = rxrpc_get_connection_maybe(conn, rxrpc_conn_get_call_input);
        if (conn) {
                rcu_read_unlock();
                ret = rxrpc_input_packet_on_conn(conn, &peer_srx, skb);
                rxrpc_put_connection(conn, rxrpc_conn_put_call_input);
                return ret;
        }

        peer = rxrpc_get_peer_maybe(peer, rxrpc_peer_get_input);
        rcu_read_unlock();

        ret = rxrpc_new_incoming_call(local, peer, NULL, &peer_srx, skb);
        rxrpc_put_peer(peer, rxrpc_peer_put_input);
        return ret;
}

/*
 * Deal with a packet that's associated with an extant connection.
 */
static int rxrpc_input_packet_on_conn(struct rxrpc_connection *conn,
                                      struct sockaddr_rxrpc *peer_srx,
                                      struct sk_buff *skb)
{
        struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
        struct rxrpc_channel *chan;
        struct rxrpc_call *call = NULL;
        unsigned int channel;

        if (sp->hdr.securityIndex != conn->security_ix)
                return rxrpc_direct_abort(skb, rxrpc_eproto_wrong_security,
                                          RXKADINCONSISTENCY, -EBADMSG);

        if (sp->hdr.serviceId != conn->service_id) {
                int old_id;

                if (!test_bit(RXRPC_CONN_PROBING_FOR_UPGRADE, &conn->flags))
                        return rxrpc_protocol_error(skb, rxrpc_eproto_reupgrade);

                old_id = cmpxchg(&conn->service_id, conn->orig_service_id,
                                 sp->hdr.serviceId);
                if (old_id != conn->orig_service_id &&
                    old_id != sp->hdr.serviceId)
                        return rxrpc_protocol_error(skb, rxrpc_eproto_bad_upgrade);
        }

        if (after(sp->hdr.serial, conn->hi_serial))
                conn->hi_serial = sp->hdr.serial;

        /* It's a connection-level packet if the call number is 0. */
        if (sp->hdr.callNumber == 0)
                return rxrpc_input_conn_packet(conn, skb);

        /* Deal with path MTU discovery probing. */
        if (sp->hdr.type == RXRPC_PACKET_TYPE_ACK &&
            conn->pmtud_probe &&
            after_eq(sp->ack.acked_serial, conn->pmtud_probe))
                rxrpc_input_probe_for_pmtud(conn, sp->ack.acked_serial, false);

        /* Call-bound packets are routed by connection channel. */
        channel = sp->hdr.cid & RXRPC_CHANNELMASK;
        chan = &conn->channels[channel];

        /* Ignore really old calls */
        if (sp->hdr.callNumber < chan->last_call)
                return just_discard;

        if (sp->hdr.callNumber == chan->last_call) {
                if (chan->call ||
                    sp->hdr.type == RXRPC_PACKET_TYPE_ABORT)
                        return just_discard;

                /* For the previous service call, if completed successfully, we
                 * discard all further packets.
                 */
                if (rxrpc_conn_is_service(conn) &&
                    chan->last_type == RXRPC_PACKET_TYPE_ACK)
                        return just_discard;

                /* But otherwise we need to retransmit the final packet from
                 * data cached in the connection record.
                 */
                if (sp->hdr.type == RXRPC_PACKET_TYPE_DATA)
                        trace_rxrpc_rx_data(chan->call_debug_id,
                                            sp->hdr.seq,
                                            sp->hdr.serial,
                                            sp->hdr.flags);
                rxrpc_conn_retransmit_call(conn, skb, channel);
                return just_discard;
        }

        call = rxrpc_try_get_call(chan->call, rxrpc_call_get_input);

        if (sp->hdr.callNumber > chan->call_id) {
                if (rxrpc_to_client(sp)) {
                        if (call)
                                rxrpc_put_call(call, rxrpc_call_put_input);
                        return rxrpc_protocol_error(skb,
                                                    rxrpc_eproto_unexpected_implicit_end);
                }

                if (call) {
                        rxrpc_implicit_end_call(call, skb);
                        rxrpc_put_call(call, rxrpc_call_put_input);
                        call = NULL;
                }
        }

        if (!call) {
                if (rxrpc_to_client(sp))
                        return rxrpc_protocol_error(skb, rxrpc_eproto_no_client_call);
                return rxrpc_new_incoming_call(conn->local, conn->peer, conn,
                                               peer_srx, skb);
        }

        rxrpc_queue_rx_call_packet(call, skb);
        rxrpc_put_call(call, rxrpc_call_put_input);
        return true;
}

/*
 * I/O and event handling thread.
 */
int rxrpc_io_thread(void *data)
{
        struct rxrpc_connection *conn;
        struct sk_buff_head rx_queue;
        struct rxrpc_local *local = data;
        struct rxrpc_call *call;
        struct sk_buff *skb;
#ifdef CONFIG_AF_RXRPC_INJECT_RX_DELAY
        ktime_t now;
#endif
        bool should_stop;
        LIST_HEAD(conn_attend_q);
        LIST_HEAD(call_attend_q);

        complete(&local->io_thread_ready);

        skb_queue_head_init(&rx_queue);

        set_user_nice(current, MIN_NICE);

        for (;;) {
                rxrpc_inc_stat(local->rxnet, stat_io_loop);

                /* Inject a delay into packets if requested. */
#ifdef CONFIG_AF_RXRPC_INJECT_RX_DELAY
                now = ktime_get_real();
                while ((skb = skb_peek(&local->rx_delay_queue))) {
                        if (ktime_before(now, skb->tstamp))
                                break;
                        skb = skb_dequeue(&local->rx_delay_queue);
                        skb_queue_tail(&local->rx_queue, skb);
                }
#endif

                if (!skb_queue_empty(&local->rx_queue)) {
                        spin_lock_irq(&local->rx_queue.lock);
                        skb_queue_splice_tail_init(&local->rx_queue, &rx_queue);
                        spin_unlock_irq(&local->rx_queue.lock);
                        trace_rxrpc_iothread_rx(local, skb_queue_len(&rx_queue));
                }

                /* Distribute packets and errors. */
                while ((skb = __skb_dequeue(&rx_queue))) {
                        struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
                        switch (skb->mark) {
                        case RXRPC_SKB_MARK_PACKET:
                                skb->priority = 0;
                                if (!rxrpc_input_packet(local, skb))
                                        rxrpc_reject_packet(local, skb);
                                trace_rxrpc_rx_done(skb->mark, skb->priority);
                                rxrpc_free_skb(skb, rxrpc_skb_put_input);
                                break;
                        case RXRPC_SKB_MARK_ERROR:
                                rxrpc_input_error(local, skb);
                                rxrpc_free_skb(skb, rxrpc_skb_put_error_report);
                                break;
                        case RXRPC_SKB_MARK_SERVICE_CONN_SECURED:
                                rxrpc_input_conn_event(sp->poke_conn, skb);
                                rxrpc_put_connection(sp->poke_conn, rxrpc_conn_put_poke);
                                rxrpc_free_skb(skb, rxrpc_skb_put_conn_secured);
                                break;
                        default:
                                WARN_ON_ONCE(1);
                                rxrpc_free_skb(skb, rxrpc_skb_put_unknown);
                                break;
                        }
                }

                /* Deal with connections that want immediate attention. */
                if (!list_empty_careful(&local->conn_attend_q)) {
                        spin_lock_irq(&local->lock);
                        list_splice_tail_init(&local->conn_attend_q, &conn_attend_q);
                        spin_unlock_irq(&local->lock);
                }

                while ((conn = list_first_entry_or_null(&conn_attend_q,
                                                        struct rxrpc_connection,
                                                        attend_link))) {
                        spin_lock_irq(&local->lock);
                        list_del_init(&conn->attend_link);
                        spin_unlock_irq(&local->lock);
                        rxrpc_input_conn_event(conn, NULL);
                        rxrpc_put_connection(conn, rxrpc_conn_put_poke);
                }

                if (test_and_clear_bit(RXRPC_CLIENT_CONN_REAP_TIMER,
                                       &local->client_conn_flags))
                        rxrpc_discard_expired_client_conns(local);

                /* Deal with calls that want immediate attention. */
                spin_lock_irq(&local->lock);
                list_splice_tail_init(&local->call_attend_q, &call_attend_q);
                spin_unlock_irq(&local->lock);

                while ((call = list_first_entry_or_null(&call_attend_q,
                                                        struct rxrpc_call,
                                                        attend_link))) {
                        spin_lock_irq(&local->lock);
                        list_del_init(&call->attend_link);
                        spin_unlock_irq(&local->lock);
                        trace_rxrpc_call_poked(call);
                        rxrpc_input_call_event(call);
                        rxrpc_put_call(call, rxrpc_call_put_poke);
                }

                if (!list_empty(&local->new_client_calls))
                        rxrpc_connect_client_calls(local);

                set_current_state(TASK_INTERRUPTIBLE);
                should_stop = kthread_should_stop();
                if (!skb_queue_empty(&local->rx_queue) ||
                    !list_empty(&local->call_attend_q) ||
                    !list_empty(&local->conn_attend_q) ||
                    !list_empty(&local->new_client_calls) ||
                    test_bit(RXRPC_CLIENT_CONN_REAP_TIMER,
                             &local->client_conn_flags)) {
                        __set_current_state(TASK_RUNNING);
                        continue;
                }

                if (should_stop)
                        break;

#ifdef CONFIG_AF_RXRPC_INJECT_RX_DELAY
                skb = skb_peek(&local->rx_delay_queue);
                if (skb) {
                        unsigned long timeout;
                        ktime_t tstamp = skb->tstamp;
                        ktime_t now = ktime_get_real();
                        s64 delay_ns = ktime_to_ns(ktime_sub(tstamp, now));

                        if (delay_ns <= 0) {
                                __set_current_state(TASK_RUNNING);
                                continue;
                        }

                        timeout = nsecs_to_jiffies(delay_ns);
                        timeout = umax(timeout, 1);
                        schedule_timeout(timeout);
                        __set_current_state(TASK_RUNNING);
                        continue;
                }
#endif

                schedule();
        }

        __set_current_state(TASK_RUNNING);
        rxrpc_see_local(local, rxrpc_local_stop);
        rxrpc_destroy_local(local);
        WRITE_ONCE(local->io_thread, NULL);
        rxrpc_see_local(local, rxrpc_local_stopped);
        return 0;
}
















   25 
































   14 




 4093 




   28 




































  296 








   25 








   84 
























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM filemap

#if !defined(_TRACE_FILEMAP_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_FILEMAP_H

#include <linux/types.h>
#include <linux/tracepoint.h>
#include <linux/mm.h>
#include <linux/memcontrol.h>
#include <linux/device.h>
#include <linux/kdev_t.h>
#include <linux/errseq.h>

DECLARE_EVENT_CLASS(mm_filemap_op_page_cache,

        TP_PROTO(struct folio *folio),

        TP_ARGS(folio),

        TP_STRUCT__entry(
                __field(u64, i_ino)
                __field(unsigned long, pfn)
                __field(unsigned long, index)
                __field(dev_t, s_dev)
                __field(unsigned char, order)
        ),

        TP_fast_assign(
                __entry->pfn = folio_pfn(folio);
                __entry->i_ino = folio->mapping->host->i_ino;
                __entry->index = folio->index;
                if (folio->mapping->host->i_sb)
                        __entry->s_dev = folio->mapping->host->i_sb->s_dev;
                else
                        __entry->s_dev = folio->mapping->host->i_rdev;
                __entry->order = folio_order(folio);
        ),

        TP_printk("dev %d:%d ino %llx pfn=0x%lx ofs=%lu order=%u",
                MAJOR(__entry->s_dev), MINOR(__entry->s_dev),
                __entry->i_ino,
                __entry->pfn,
                __entry->index << PAGE_SHIFT,
                __entry->order)
);

DEFINE_EVENT(mm_filemap_op_page_cache, mm_filemap_delete_from_page_cache,
        TP_PROTO(struct folio *folio),
        TP_ARGS(folio)
        );

DEFINE_EVENT(mm_filemap_op_page_cache, mm_filemap_add_to_page_cache,
        TP_PROTO(struct folio *folio),
        TP_ARGS(folio)
        );

DECLARE_EVENT_CLASS(mm_filemap_op_page_cache_range,

        TP_PROTO(
                struct address_space *mapping,
                pgoff_t index,
                pgoff_t last_index
        ),

        TP_ARGS(mapping, index, last_index),

        TP_STRUCT__entry(
                __field(u64, i_ino)
                __field(dev_t, s_dev)
                __field(unsigned long, index)
                __field(unsigned long, last_index)
        ),

        TP_fast_assign(
                __entry->i_ino = mapping->host->i_ino;
                if (mapping->host->i_sb)
                        __entry->s_dev =
                                mapping->host->i_sb->s_dev;
                else
                        __entry->s_dev = mapping->host->i_rdev;
                __entry->index = index;
                __entry->last_index = last_index;
        ),

        TP_printk(
                "dev=%d:%d ino=%llx ofs=%lld-%lld",
                MAJOR(__entry->s_dev),
                MINOR(__entry->s_dev), __entry->i_ino,
                ((loff_t)__entry->index) << PAGE_SHIFT,
                ((((loff_t)__entry->last_index + 1) << PAGE_SHIFT) - 1)
        )
);

DEFINE_EVENT(mm_filemap_op_page_cache_range, mm_filemap_get_pages,
        TP_PROTO(
                struct address_space *mapping,
                pgoff_t index,
                pgoff_t last_index
        ),
        TP_ARGS(mapping, index, last_index)
);

DEFINE_EVENT(mm_filemap_op_page_cache_range, mm_filemap_map_pages,
        TP_PROTO(
                struct address_space *mapping,
                pgoff_t index,
                pgoff_t last_index
        ),
        TP_ARGS(mapping, index, last_index)
);

TRACE_EVENT(mm_filemap_fault,
        TP_PROTO(struct address_space *mapping, pgoff_t index),

        TP_ARGS(mapping, index),

        TP_STRUCT__entry(
                __field(u64, i_ino)
                __field(dev_t, s_dev)
                __field(unsigned long, index)
        ),

        TP_fast_assign(
                __entry->i_ino = mapping->host->i_ino;
                if (mapping->host->i_sb)
                        __entry->s_dev =
                                mapping->host->i_sb->s_dev;
                else
                        __entry->s_dev = mapping->host->i_rdev;
                __entry->index = index;
        ),

        TP_printk(
                "dev=%d:%d ino=%llx ofs=%lld",
                MAJOR(__entry->s_dev),
                MINOR(__entry->s_dev), __entry->i_ino,
                ((loff_t)__entry->index) << PAGE_SHIFT
        )
);

TRACE_EVENT(filemap_set_wb_err,
                TP_PROTO(struct address_space *mapping, errseq_t eseq),

                TP_ARGS(mapping, eseq),

                TP_STRUCT__entry(
                        __field(u64, i_ino)
                        __field(dev_t, s_dev)
                        __field(errseq_t, errseq)
                ),

                TP_fast_assign(
                        __entry->i_ino = mapping->host->i_ino;
                        __entry->errseq = eseq;
                        if (mapping->host->i_sb)
                                __entry->s_dev = mapping->host->i_sb->s_dev;
                        else
                                __entry->s_dev = mapping->host->i_rdev;
                ),

                TP_printk("dev=%d:%d ino=0x%llx errseq=0x%x",
                        MAJOR(__entry->s_dev), MINOR(__entry->s_dev),
                        __entry->i_ino, __entry->errseq)
);

TRACE_EVENT(file_check_and_advance_wb_err,
                TP_PROTO(struct file *file, errseq_t old),

                TP_ARGS(file, old),

                TP_STRUCT__entry(
                        __field(u64, i_ino)
                        __field(struct file *, file)
                        __field(dev_t, s_dev)
                        __field(errseq_t, old)
                        __field(errseq_t, new)
                ),

                TP_fast_assign(
                        __entry->file = file;
                        __entry->i_ino = file->f_mapping->host->i_ino;
                        if (file->f_mapping->host->i_sb)
                                __entry->s_dev =
                                        file->f_mapping->host->i_sb->s_dev;
                        else
                                __entry->s_dev =
                                        file->f_mapping->host->i_rdev;
                        __entry->old = old;
                        __entry->new = file->f_wb_err;
                ),

                TP_printk("file=%p dev=%d:%d ino=0x%llx old=0x%x new=0x%x",
                        __entry->file, MAJOR(__entry->s_dev),
                        MINOR(__entry->s_dev), __entry->i_ino, __entry->old,
                        __entry->new)
);
#endif /* _TRACE_FILEMAP_H */

/* This part must be outside protection */
#include <trace/define_trace.h>


























































  169 


  492 
























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                Checksumming functions for IPv6
 *
 * Authors:        Jorge Cwik, <jorge@laser.satlink.net>
 *                Arnt Gulbrandsen, <agulbra@nvg.unit.no>
 *                Borrows very liberally from tcp.c and ip.c, see those
 *                files for more names.
 */

/*
 *        Fixes:
 *
 *        Ralf Baechle                        :        generic ipv6 checksum
 *        <ralf@waldorf-gmbh.de>
 */

#ifndef _CHECKSUM_IPV6_H
#define _CHECKSUM_IPV6_H

#include <asm/types.h>
#include <asm/byteorder.h>
#include <net/ip.h>
#include <asm/checksum.h>
#include <linux/in6.h>
#include <linux/tcp.h>
#include <linux/ipv6.h>

#ifndef _HAVE_ARCH_IPV6_CSUM
__sum16 csum_ipv6_magic(const struct in6_addr *saddr,
                        const struct in6_addr *daddr,
                        __u32 len, __u8 proto, __wsum csum);
#endif

static inline __wsum ip6_compute_pseudo(struct sk_buff *skb, int proto)
{
        return ~csum_unfold(csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
                                            &ipv6_hdr(skb)->daddr,
                                            skb->len, proto, 0));
}

static __inline__ __sum16 tcp_v6_check(int len,
                                   const struct in6_addr *saddr,
                                   const struct in6_addr *daddr,
                                   __wsum base)
{
        return csum_ipv6_magic(saddr, daddr, len, IPPROTO_TCP, base);
}

static inline void __tcp_v6_send_check(struct sk_buff *skb,
                                       const struct in6_addr *saddr,
                                       const struct in6_addr *daddr)
{
        struct tcphdr *th = tcp_hdr(skb);

        th->check = ~tcp_v6_check(skb->len, saddr, daddr, 0);
        skb->csum_start = skb_transport_header(skb) - skb->head;
        skb->csum_offset = offsetof(struct tcphdr, check);
}

static inline void tcp_v6_gso_csum_prep(struct sk_buff *skb)
{
        struct ipv6hdr *ipv6h = ipv6_hdr(skb);
        struct tcphdr *th = tcp_hdr(skb);

        ipv6h->payload_len = 0;
        th->check = ~tcp_v6_check(0, &ipv6h->saddr, &ipv6h->daddr, 0);
}

static inline __sum16 udp_v6_check(int len,
                                   const struct in6_addr *saddr,
                                   const struct in6_addr *daddr,
                                   __wsum base)
{
        return csum_ipv6_magic(saddr, daddr, len, IPPROTO_UDP, base);
}

void udp6_set_csum(bool nocheck, struct sk_buff *skb,
                   const struct in6_addr *saddr,
                   const struct in6_addr *daddr, int len);
#endif




































 2366 



 2366 

 2366 





 1607 

















































    6 

    6 
    6 





 1822 




    7 
    7 














 1607 






 1607 

    6 



 1607 


 1607 







 1822 
 1822 
    4 
 1822 
 1822 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
// SPDX-License-Identifier: GPL-2.0-or-later

/*
 * Functions for initializing, allocating, freeing and duplicating VMAs. Shared
 * between CONFIG_MMU and non-CONFIG_MMU kernel configurations.
 */

/*
 * To allow for userland testing we place internal dependencies in
 * vma_internal.h and external VMA API declarations in vma.h.
 */
#include "vma_internal.h"
#include "vma.h"

/* SLAB cache for vm_area_struct structures */
static struct kmem_cache *vm_area_cachep;

void __init vma_state_init(void)
{
        struct kmem_cache_args args = {
                .use_freeptr_offset = true,
                .freeptr_offset = offsetof(struct vm_area_struct, vm_freeptr),
                .sheaf_capacity = 32,
        };

        vm_area_cachep = kmem_cache_create("vm_area_struct",
                        sizeof(struct vm_area_struct), &args,
                        SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU|
                        SLAB_ACCOUNT);
}

struct vm_area_struct *vm_area_alloc(struct mm_struct *mm)
{
        struct vm_area_struct *vma;

        vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
        if (!vma)
                return NULL;

        vma_init(vma, mm);

        return vma;
}

static void vm_area_init_from(const struct vm_area_struct *src,
                              struct vm_area_struct *dest)
{
        dest->vm_mm = src->vm_mm;
        dest->vm_ops = src->vm_ops;
        dest->vm_start = src->vm_start;
        dest->vm_end = src->vm_end;
        dest->anon_vma = src->anon_vma;
        dest->vm_pgoff = vma_start_pgoff(src);
        __vma_set_anon_pgoff(dest, vma_start_anon_pgoff(src));
        dest->vm_file = src->vm_file;
        dest->vm_private_data = src->vm_private_data;
        vm_flags_init(dest, src->vm_flags);
        memcpy(&dest->vm_page_prot, &src->vm_page_prot,
               sizeof(dest->vm_page_prot));
        /*
         * src->shared.rb may be modified concurrently when called from
         * dup_mmap(), but the clone will reinitialize it.
         */
        data_race(memcpy(&dest->shared, &src->shared, sizeof(dest->shared)));
        memcpy(&dest->vm_userfaultfd_ctx, &src->vm_userfaultfd_ctx,
               sizeof(dest->vm_userfaultfd_ctx));
#ifdef CONFIG_ANON_VMA_NAME
        dest->anon_name = src->anon_name;
#endif
#ifdef CONFIG_SWAP
        memcpy(&dest->swap_readahead_info, &src->swap_readahead_info,
               sizeof(dest->swap_readahead_info));
#endif
#ifndef CONFIG_MMU
        dest->vm_region = src->vm_region;
#endif
#ifdef CONFIG_NUMA
        dest->vm_policy = src->vm_policy;
#endif
#ifdef __HAVE_PFNMAP_TRACKING
        dest->pfnmap_track_ctx = NULL;
#endif
}

#ifdef __HAVE_PFNMAP_TRACKING
static inline int vma_pfnmap_track_ctx_dup(struct vm_area_struct *orig,
                struct vm_area_struct *new)
{
        struct pfnmap_track_ctx *ctx = orig->pfnmap_track_ctx;

        if (likely(!ctx))
                return 0;

        /*
         * We don't expect to ever hit this. If ever required, we would have
         * to duplicate the tracking.
         */
        if (unlikely(kref_read(&ctx->kref) >= REFCOUNT_MAX))
                return -ENOMEM;
        kref_get(&ctx->kref);
        new->pfnmap_track_ctx = ctx;
        return 0;
}

static inline void vma_pfnmap_track_ctx_release(struct vm_area_struct *vma)
{
        struct pfnmap_track_ctx *ctx = vma->pfnmap_track_ctx;

        if (likely(!ctx))
                return;

        kref_put(&ctx->kref, pfnmap_track_ctx_release);
        vma->pfnmap_track_ctx = NULL;
}
#else
static inline int vma_pfnmap_track_ctx_dup(struct vm_area_struct *orig,
                struct vm_area_struct *new)
{
        return 0;
}
static inline void vma_pfnmap_track_ctx_release(struct vm_area_struct *vma)
{
}
#endif

struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig)
{
        struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);

        if (!new)
                return NULL;

        ASSERT_EXCLUSIVE_WRITER(orig->vm_flags);
        ASSERT_EXCLUSIVE_WRITER(orig->vm_file);
        vm_area_init_from(orig, new);

        if (vma_pfnmap_track_ctx_dup(orig, new)) {
                kmem_cache_free(vm_area_cachep, new);
                return NULL;
        }
        vma_lock_init(new, true);
        INIT_LIST_HEAD(&new->anon_vma_chain);
        vma_numab_state_init(new);
        dup_anon_vma_name(orig, new);

        return new;
}

void vm_area_free(struct vm_area_struct *vma)
{
        /* The vma should be detached while being destroyed. */
        vma_assert_detached(vma);
        vma_numab_state_free(vma);
        free_anon_vma_name(vma);
        vma_pfnmap_track_ctx_release(vma);
        kmem_cache_free(vm_area_cachep, vma);
}






















   14 



   13 


   13 
   12 
    7 

    1 

    1 


   12 





   34 



   34 


   34 











   34 
   31 

   14 

   14 



   14 








    2 

    1 



    2 







   17 



   17 


   16 
   12 



   16 
   13 
    2 




   16 
   10 
    9 





    9 



   16 



    2 




























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *   Copyright (C) International Business Machines Corp., 2000-2002
 *   Portions Copyright (C) Christoph Hellwig, 2001-2002
 */

#include <linux/mm.h>
#include <linux/fs.h>
#include <linux/filelock.h>
#include <linux/posix_acl.h>
#include <linux/quotaops.h>
#include "jfs_incore.h"
#include "jfs_inode.h"
#include "jfs_dmap.h"
#include "jfs_txnmgr.h"
#include "jfs_xattr.h"
#include "jfs_acl.h"
#include "jfs_debug.h"

int jfs_fsync(struct file *file, loff_t start, loff_t end, int datasync)
{
        struct inode *inode = file->f_mapping->host;
        int rc = 0;

        rc = file_write_and_wait_range(file, start, end);
        if (rc)
                return rc;

        inode_lock(inode);
        if (!(inode_state_read_once(inode) & I_DIRTY_ALL) ||
            (datasync && !(inode_state_read_once(inode) & I_DIRTY_DATASYNC))) {
                /* Make sure committed changes hit the disk */
                jfs_flush_journal(JFS_SBI(inode->i_sb)->log, 1);
                inode_unlock(inode);
                return rc;
        }

        rc |= jfs_commit_inode(inode, 1);
        inode_unlock(inode);

        return rc ? -EIO : 0;
}

static int jfs_open(struct inode *inode, struct file *file)
{
        int rc;

        if (S_ISREG(inode->i_mode) && inode->i_size < 0)
                return -EIO;

        if ((rc = dquot_file_open(inode, file)))
                return rc;

        /*
         * We attempt to allow only one "active" file open per aggregate
         * group.  Otherwise, appending to files in parallel can cause
         * fragmentation within the files.
         *
         * If the file is empty, it was probably just created and going
         * to be written to.  If it has a size, we'll hold off until the
         * file is actually grown.
         */
        if (S_ISREG(inode->i_mode) && file->f_mode & FMODE_WRITE &&
            (inode->i_size == 0)) {
                struct jfs_inode_info *ji = JFS_IP(inode);
                spin_lock_irq(&ji->ag_lock);
                if (ji->active_ag == -1) {
                        struct jfs_sb_info *jfs_sb = JFS_SBI(inode->i_sb);
                        ji->active_ag = BLKTOAG(addressPXD(&ji->ixpxd), jfs_sb);
                        atomic_inc(&jfs_sb->bmap->db_active[ji->active_ag]);
                }
                spin_unlock_irq(&ji->ag_lock);
        }

        return 0;
}
static int jfs_release(struct inode *inode, struct file *file)
{
        struct jfs_inode_info *ji = JFS_IP(inode);

        spin_lock_irq(&ji->ag_lock);
        if (ji->active_ag != -1) {
                struct bmap *bmap = JFS_SBI(inode->i_sb)->bmap;
                atomic_dec(&bmap->db_active[ji->active_ag]);
                ji->active_ag = -1;
        }
        spin_unlock_irq(&ji->ag_lock);

        return 0;
}

int jfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
                struct iattr *iattr)
{
        struct inode *inode = d_inode(dentry);
        int rc;

        rc = setattr_prepare(&nop_mnt_idmap, dentry, iattr);
        if (rc)
                return rc;

        if (is_quota_modification(&nop_mnt_idmap, inode, iattr)) {
                rc = dquot_initialize(inode);
                if (rc)
                        return rc;
        }
        if ((iattr->ia_valid & ATTR_UID && !uid_eq(iattr->ia_uid, inode->i_uid)) ||
            (iattr->ia_valid & ATTR_GID && !gid_eq(iattr->ia_gid, inode->i_gid))) {
                rc = dquot_transfer(&nop_mnt_idmap, inode, iattr);
                if (rc)
                        return rc;
        }

        if ((iattr->ia_valid & ATTR_SIZE) &&
            iattr->ia_size != i_size_read(inode)) {
                inode_dio_wait(inode);

                rc = inode_newsize_ok(inode, iattr->ia_size);
                if (rc)
                        return rc;

                truncate_setsize(inode, iattr->ia_size);
                jfs_truncate(inode);
        }

        setattr_copy(&nop_mnt_idmap, inode, iattr);
        mark_inode_dirty(inode);

        if (iattr->ia_valid & ATTR_MODE)
                rc = posix_acl_chmod(&nop_mnt_idmap, dentry, inode->i_mode);
        return rc;
}

const struct inode_operations jfs_file_inode_operations = {
        .listxattr        = jfs_listxattr,
        .setattr        = jfs_setattr,
        .fileattr_get        = jfs_fileattr_get,
        .fileattr_set        = jfs_fileattr_set,
#ifdef CONFIG_JFS_POSIX_ACL
        .get_inode_acl        = jfs_get_acl,
        .set_acl        = jfs_set_acl,
#endif
};

const struct file_operations jfs_file_operations = {
        .open                = jfs_open,
        .llseek                = generic_file_llseek,
        .read_iter        = generic_file_read_iter,
        .write_iter        = generic_file_write_iter,
        .mmap_prepare        = generic_file_mmap_prepare,
        .splice_read        = filemap_splice_read,
        .splice_write        = iter_file_splice_write,
        .fsync                = jfs_fsync,
        .release        = jfs_release,
        .unlocked_ioctl = jfs_ioctl,
        .compat_ioctl        = compat_ptr_ioctl,
        .setlease        = generic_setlease,
};







































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































 1092 
 1092 






















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   61 



   61 







































































































































































































  123 












  123 












































  123 


  123 
  123 

  123 
  124 




























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
// SPDX-License-Identifier: GPL-2.0-or-later
/* auditsc.c -- System-call auditing support
 * Handles all system-call specific auditing features.
 *
 * Copyright 2003-2004 Red Hat Inc., Durham, North Carolina.
 * Copyright 2005 Hewlett-Packard Development Company, L.P.
 * Copyright (C) 2005, 2006 IBM Corporation
 * All Rights Reserved.
 *
 * Written by Rickard E. (Rik) Faith <faith@redhat.com>
 *
 * Many of the ideas implemented here are from Stephen C. Tweedie,
 * especially the idea of avoiding a copy by using getname.
 *
 * The method for actual interception of syscall entry and exit (not in
 * this file -- see entry.S) is based on a GPL'd patch written by
 * okir@suse.de and Copyright 2003 SuSE Linux AG.
 *
 * POSIX message queue support added by George Wilson <ltcgcw@us.ibm.com>,
 * 2006.
 *
 * The support of additional filter rules compares (>, <, >=, <=) was
 * added by Dustin Kirkland <dustin.kirkland@us.ibm.com>, 2005.
 *
 * Modified by Amy Griffis <amy.griffis@hp.com> to collect additional
 * filesystem information.
 *
 * Subject and object context labeling support added by <danjones@us.ibm.com>
 * and <dustin.kirkland@us.ibm.com> for LSPP certification compliance.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/init.h>
#include <asm/types.h>
#include <linux/atomic.h>
#include <linux/fs.h>
#include <linux/namei.h>
#include <linux/mm.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/mount.h>
#include <linux/socket.h>
#include <linux/mqueue.h>
#include <linux/audit.h>
#include <linux/personality.h>
#include <linux/time.h>
#include <linux/netlink.h>
#include <linux/compiler.h>
#include <asm/unistd.h>
#include <linux/security.h>
#include <linux/list.h>
#include <linux/binfmts.h>
#include <linux/highmem.h>
#include <linux/syscalls.h>
#include <asm/syscall.h>
#include <linux/capability.h>
#include <linux/fs_struct.h>
#include <linux/compat.h>
#include <linux/ctype.h>
#include <linux/string.h>
#include <linux/uaccess.h>
#include <linux/fsnotify_backend.h>
#include <uapi/linux/limits.h>
#include <uapi/linux/netfilter/nf_tables.h>
#include <uapi/linux/openat2.h> // struct open_how
#include <uapi/linux/fanotify.h>

#include "audit.h"

/* flags stating the success for a syscall */
#define AUDITSC_INVALID 0
#define AUDITSC_SUCCESS 1
#define AUDITSC_FAILURE 2

/* no execve audit message should be longer than this (userspace limits),
 * see the note near the top of audit_log_execve_info() about this value */
#define MAX_EXECVE_AUDIT_LEN 7500

/* max length to print of cmdline/proctitle value during audit */
#define MAX_PROCTITLE_AUDIT_LEN 128

/* number of audit rules */
int audit_n_rules;

/* determines whether we collect data for signals sent */
int audit_signals;

struct audit_aux_data {
        struct audit_aux_data        *next;
        int                        type;
};

/* Number of target pids per aux struct. */
#define AUDIT_AUX_PIDS        16

struct audit_aux_data_pids {
        struct audit_aux_data        d;
        pid_t                        target_pid[AUDIT_AUX_PIDS];
        kuid_t                        target_auid[AUDIT_AUX_PIDS];
        kuid_t                        target_uid[AUDIT_AUX_PIDS];
        unsigned int                target_sessionid[AUDIT_AUX_PIDS];
        struct lsm_prop                target_ref[AUDIT_AUX_PIDS];
        char                         target_comm[AUDIT_AUX_PIDS][TASK_COMM_LEN];
        int                        pid_count;
};

struct audit_aux_data_bprm_fcaps {
        struct audit_aux_data        d;
        struct audit_cap_data        fcap;
        unsigned int                fcap_ver;
        struct audit_cap_data        old_pcap;
        struct audit_cap_data        new_pcap;
};

struct audit_tree_refs {
        struct audit_tree_refs *next;
        struct audit_chunk *c[31];
};

struct audit_nfcfgop_tab {
        enum audit_nfcfgop        op;
        const char                *s;
};

static const struct audit_nfcfgop_tab audit_nfcfgs[] = {
        { AUDIT_XT_OP_REGISTER,                        "xt_register"                   },
        { AUDIT_XT_OP_REPLACE,                        "xt_replace"                   },
        { AUDIT_XT_OP_UNREGISTER,                "xt_unregister"                   },
        { AUDIT_NFT_OP_TABLE_REGISTER,                "nft_register_table"           },
        { AUDIT_NFT_OP_TABLE_UNREGISTER,        "nft_unregister_table"           },
        { AUDIT_NFT_OP_CHAIN_REGISTER,                "nft_register_chain"           },
        { AUDIT_NFT_OP_CHAIN_UNREGISTER,        "nft_unregister_chain"           },
        { AUDIT_NFT_OP_RULE_REGISTER,                "nft_register_rule"           },
        { AUDIT_NFT_OP_RULE_UNREGISTER,                "nft_unregister_rule"           },
        { AUDIT_NFT_OP_SET_REGISTER,                "nft_register_set"           },
        { AUDIT_NFT_OP_SET_UNREGISTER,                "nft_unregister_set"           },
        { AUDIT_NFT_OP_SETELEM_REGISTER,        "nft_register_setelem"           },
        { AUDIT_NFT_OP_SETELEM_UNREGISTER,        "nft_unregister_setelem"   },
        { AUDIT_NFT_OP_GEN_REGISTER,                "nft_register_gen"           },
        { AUDIT_NFT_OP_OBJ_REGISTER,                "nft_register_obj"           },
        { AUDIT_NFT_OP_OBJ_UNREGISTER,                "nft_unregister_obj"           },
        { AUDIT_NFT_OP_OBJ_RESET,                "nft_reset_obj"                   },
        { AUDIT_NFT_OP_FLOWTABLE_REGISTER,        "nft_register_flowtable"   },
        { AUDIT_NFT_OP_FLOWTABLE_UNREGISTER,        "nft_unregister_flowtable" },
        { AUDIT_NFT_OP_SETELEM_RESET,                "nft_reset_setelem"        },
        { AUDIT_NFT_OP_RULE_RESET,                "nft_reset_rule"           },
        { AUDIT_NFT_OP_INVALID,                        "nft_invalid"                   },
};

static int audit_match_perm(struct audit_context *ctx, int mask)
{
        unsigned int n;

        if (unlikely(!ctx))
                return 0;
        n = ctx->major;

        switch (audit_classify_syscall(ctx->arch, n)) {
        case AUDITSC_NATIVE:
                if ((mask & AUDIT_PERM_WRITE) &&
                     audit_match_class(AUDIT_CLASS_WRITE, n))
                        return 1;
                if ((mask & AUDIT_PERM_READ) &&
                     audit_match_class(AUDIT_CLASS_READ, n))
                        return 1;
                if ((mask & AUDIT_PERM_ATTR) &&
                     audit_match_class(AUDIT_CLASS_CHATTR, n))
                        return 1;
                return 0;
        case AUDITSC_COMPAT: /* 32bit on biarch */
                if ((mask & AUDIT_PERM_WRITE) &&
                     audit_match_class(AUDIT_CLASS_WRITE_32, n))
                        return 1;
                if ((mask & AUDIT_PERM_READ) &&
                     audit_match_class(AUDIT_CLASS_READ_32, n))
                        return 1;
                if ((mask & AUDIT_PERM_ATTR) &&
                     audit_match_class(AUDIT_CLASS_CHATTR_32, n))
                        return 1;
                return 0;
        case AUDITSC_OPEN:
                return mask & ACC_MODE(ctx->argv[1]);
        case AUDITSC_OPENAT:
                return mask & ACC_MODE(ctx->argv[2]);
        case AUDITSC_SOCKETCALL:
                return ((mask & AUDIT_PERM_WRITE) && ctx->argv[0] == SYS_BIND);
        case AUDITSC_EXECVE:
                return mask & AUDIT_PERM_EXEC;
        case AUDITSC_OPENAT2:
                return mask & ACC_MODE((u32)ctx->openat2.flags);
        default:
                return 0;
        }
}

static int audit_match_filetype(struct audit_context *ctx, int val)
{
        struct audit_names *n;
        umode_t mode = (umode_t)val;

        if (unlikely(!ctx))
                return 0;

        list_for_each_entry(n, &ctx->names_list, list) {
                if ((n->ino != AUDIT_INO_UNSET) &&
                    ((n->mode & S_IFMT) == mode))
                        return 1;
        }

        return 0;
}

/*
 * We keep a linked list of fixed-sized (31 pointer) arrays of audit_chunk *;
 * ->first_trees points to its beginning, ->trees - to the current end of data.
 * ->tree_count is the number of free entries in array pointed to by ->trees.
 * Original condition is (NULL, NULL, 0); as soon as it grows we never revert to NULL,
 * "empty" becomes (p, p, 31) afterwards.  We don't shrink the list (and seriously,
 * it's going to remain 1-element for almost any setup) until we free context itself.
 * References in it _are_ dropped - at the same time we free/drop aux stuff.
 */

static void audit_set_auditable(struct audit_context *ctx)
{
        if (!ctx->prio) {
                ctx->prio = 1;
                ctx->current_state = AUDIT_STATE_RECORD;
        }
}

static int put_tree_ref(struct audit_context *ctx, struct audit_chunk *chunk)
{
        struct audit_tree_refs *p = ctx->trees;
        int left = ctx->tree_count;

        if (likely(left)) {
                p->c[--left] = chunk;
                ctx->tree_count = left;
                return 1;
        }
        if (!p)
                return 0;
        p = p->next;
        if (p) {
                p->c[30] = chunk;
                ctx->trees = p;
                ctx->tree_count = 30;
                return 1;
        }
        return 0;
}

static int grow_tree_refs(struct audit_context *ctx)
{
        struct audit_tree_refs *p = ctx->trees;

        ctx->trees = kzalloc_obj(struct audit_tree_refs);
        if (!ctx->trees) {
                ctx->trees = p;
                return 0;
        }
        if (p)
                p->next = ctx->trees;
        else
                ctx->first_trees = ctx->trees;
        ctx->tree_count = 31;
        return 1;
}

static void unroll_tree_refs(struct audit_context *ctx,
                      struct audit_tree_refs *p, int count)
{
        struct audit_tree_refs *q;
        int n;

        if (!p) {
                /* we started with empty chain */
                p = ctx->first_trees;
                count = 31;
                /* if the very first allocation has failed, nothing to do */
                if (!p)
                        return;
        }
        n = count;
        for (q = p; q != ctx->trees; q = q->next, n = 31) {
                while (n--) {
                        audit_put_chunk(q->c[n]);
                        q->c[n] = NULL;
                }
        }
        while (n-- > ctx->tree_count) {
                audit_put_chunk(q->c[n]);
                q->c[n] = NULL;
        }
        ctx->trees = p;
        ctx->tree_count = count;
}

static void free_tree_refs(struct audit_context *ctx)
{
        struct audit_tree_refs *p, *q;

        for (p = ctx->first_trees; p; p = q) {
                q = p->next;
                kfree(p);
        }
}

static int match_tree_refs(struct audit_context *ctx, struct audit_tree *tree)
{
        struct audit_tree_refs *p;
        int n;

        if (!tree)
                return 0;
        /* full ones */
        for (p = ctx->first_trees; p != ctx->trees; p = p->next) {
                for (n = 0; n < 31; n++)
                        if (audit_tree_match(p->c[n], tree))
                                return 1;
        }
        /* partial */
        if (p) {
                for (n = ctx->tree_count; n < 31; n++)
                        if (audit_tree_match(p->c[n], tree))
                                return 1;
        }
        return 0;
}

static int audit_compare_uid(kuid_t uid,
                             struct audit_names *name,
                             struct audit_field *f,
                             struct audit_context *ctx)
{
        struct audit_names *n;
        int rc;

        if (name) {
                rc = audit_uid_comparator(uid, f->op, name->uid);
                if (rc)
                        return rc;
        }

        if (ctx) {
                list_for_each_entry(n, &ctx->names_list, list) {
                        rc = audit_uid_comparator(uid, f->op, n->uid);
                        if (rc)
                                return rc;
                }
        }
        return 0;
}

static int audit_compare_gid(kgid_t gid,
                             struct audit_names *name,
                             struct audit_field *f,
                             struct audit_context *ctx)
{
        struct audit_names *n;
        int rc;

        if (name) {
                rc = audit_gid_comparator(gid, f->op, name->gid);
                if (rc)
                        return rc;
        }

        if (ctx) {
                list_for_each_entry(n, &ctx->names_list, list) {
                        rc = audit_gid_comparator(gid, f->op, n->gid);
                        if (rc)
                                return rc;
                }
        }
        return 0;
}

static int audit_field_compare(struct task_struct *tsk,
                               const struct cred *cred,
                               struct audit_field *f,
                               struct audit_context *ctx,
                               struct audit_names *name)
{
        switch (f->val) {
        /* process to file object comparisons */
        case AUDIT_COMPARE_UID_TO_OBJ_UID:
                return audit_compare_uid(cred->uid, name, f, ctx);
        case AUDIT_COMPARE_GID_TO_OBJ_GID:
                return audit_compare_gid(cred->gid, name, f, ctx);
        case AUDIT_COMPARE_EUID_TO_OBJ_UID:
                return audit_compare_uid(cred->euid, name, f, ctx);
        case AUDIT_COMPARE_EGID_TO_OBJ_GID:
                return audit_compare_gid(cred->egid, name, f, ctx);
        case AUDIT_COMPARE_AUID_TO_OBJ_UID:
                return audit_compare_uid(audit_get_loginuid(tsk), name, f, ctx);
        case AUDIT_COMPARE_SUID_TO_OBJ_UID:
                return audit_compare_uid(cred->suid, name, f, ctx);
        case AUDIT_COMPARE_SGID_TO_OBJ_GID:
                return audit_compare_gid(cred->sgid, name, f, ctx);
        case AUDIT_COMPARE_FSUID_TO_OBJ_UID:
                return audit_compare_uid(cred->fsuid, name, f, ctx);
        case AUDIT_COMPARE_FSGID_TO_OBJ_GID:
                return audit_compare_gid(cred->fsgid, name, f, ctx);
        /* uid comparisons */
        case AUDIT_COMPARE_UID_TO_AUID:
                return audit_uid_comparator(cred->uid, f->op,
                                            audit_get_loginuid(tsk));
        case AUDIT_COMPARE_UID_TO_EUID:
                return audit_uid_comparator(cred->uid, f->op, cred->euid);
        case AUDIT_COMPARE_UID_TO_SUID:
                return audit_uid_comparator(cred->uid, f->op, cred->suid);
        case AUDIT_COMPARE_UID_TO_FSUID:
                return audit_uid_comparator(cred->uid, f->op, cred->fsuid);
        /* auid comparisons */
        case AUDIT_COMPARE_AUID_TO_EUID:
                return audit_uid_comparator(audit_get_loginuid(tsk), f->op,
                                            cred->euid);
        case AUDIT_COMPARE_AUID_TO_SUID:
                return audit_uid_comparator(audit_get_loginuid(tsk), f->op,
                                            cred->suid);
        case AUDIT_COMPARE_AUID_TO_FSUID:
                return audit_uid_comparator(audit_get_loginuid(tsk), f->op,
                                            cred->fsuid);
        /* euid comparisons */
        case AUDIT_COMPARE_EUID_TO_SUID:
                return audit_uid_comparator(cred->euid, f->op, cred->suid);
        case AUDIT_COMPARE_EUID_TO_FSUID:
                return audit_uid_comparator(cred->euid, f->op, cred->fsuid);
        /* suid comparisons */
        case AUDIT_COMPARE_SUID_TO_FSUID:
                return audit_uid_comparator(cred->suid, f->op, cred->fsuid);
        /* gid comparisons */
        case AUDIT_COMPARE_GID_TO_EGID:
                return audit_gid_comparator(cred->gid, f->op, cred->egid);
        case AUDIT_COMPARE_GID_TO_SGID:
                return audit_gid_comparator(cred->gid, f->op, cred->sgid);
        case AUDIT_COMPARE_GID_TO_FSGID:
                return audit_gid_comparator(cred->gid, f->op, cred->fsgid);
        /* egid comparisons */
        case AUDIT_COMPARE_EGID_TO_SGID:
                return audit_gid_comparator(cred->egid, f->op, cred->sgid);
        case AUDIT_COMPARE_EGID_TO_FSGID:
                return audit_gid_comparator(cred->egid, f->op, cred->fsgid);
        /* sgid comparison */
        case AUDIT_COMPARE_SGID_TO_FSGID:
                return audit_gid_comparator(cred->sgid, f->op, cred->fsgid);
        default:
                WARN(1, "Missing AUDIT_COMPARE define.  Report as a bug\n");
                return 0;
        }
        return 0;
}

/* Determine if any context name data matches a rule's watch data */
/* Compare a task_struct with an audit_rule.  Return 1 on match, 0
 * otherwise.
 *
 * If task_creation is true, this is an explicit indication that we are
 * filtering a task rule at task creation time.  This and tsk == current are
 * the only situations where tsk->cred may be accessed without an rcu read lock.
 */
static int audit_filter_rules(struct task_struct *tsk,
                              struct audit_krule *rule,
                              struct audit_context *ctx,
                              struct audit_names *name,
                              enum audit_state *state,
                              bool task_creation)
{
        const struct cred *cred;
        int i, need_sid = 1;
        struct lsm_prop prop = { };
        unsigned int sessionid;

        if (ctx && rule->prio <= ctx->prio)
                return 0;

        cred = rcu_dereference_check(tsk->cred, tsk == current || task_creation);

        for (i = 0; i < rule->field_count; i++) {
                struct audit_field *f = &rule->fields[i];
                struct audit_names *n;
                int result = 0;
                pid_t pid;

                switch (f->type) {
                case AUDIT_PID:
                        pid = task_tgid_nr(tsk);
                        result = audit_comparator(pid, f->op, f->val);
                        break;
                case AUDIT_PPID:
                        if (ctx) {
                                if (!ctx->ppid)
                                        ctx->ppid = task_ppid_nr(tsk);
                                result = audit_comparator(ctx->ppid, f->op, f->val);
                        }
                        break;
                case AUDIT_EXE:
                        result = audit_exe_compare(tsk, rule->exe);
                        if (f->op == Audit_not_equal)
                                result = !result;
                        break;
                case AUDIT_UID:
                        result = audit_uid_comparator(cred->uid, f->op, f->uid);
                        break;
                case AUDIT_EUID:
                        result = audit_uid_comparator(cred->euid, f->op, f->uid);
                        break;
                case AUDIT_SUID:
                        result = audit_uid_comparator(cred->suid, f->op, f->uid);
                        break;
                case AUDIT_FSUID:
                        result = audit_uid_comparator(cred->fsuid, f->op, f->uid);
                        break;
                case AUDIT_GID:
                        result = audit_gid_comparator(cred->gid, f->op, f->gid);
                        if (f->op == Audit_equal) {
                                if (!result)
                                        result = groups_search(cred->group_info, f->gid);
                        } else if (f->op == Audit_not_equal) {
                                if (result)
                                        result = !groups_search(cred->group_info, f->gid);
                        }
                        break;
                case AUDIT_EGID:
                        result = audit_gid_comparator(cred->egid, f->op, f->gid);
                        if (f->op == Audit_equal) {
                                if (!result)
                                        result = groups_search(cred->group_info, f->gid);
                        } else if (f->op == Audit_not_equal) {
                                if (result)
                                        result = !groups_search(cred->group_info, f->gid);
                        }
                        break;
                case AUDIT_SGID:
                        result = audit_gid_comparator(cred->sgid, f->op, f->gid);
                        break;
                case AUDIT_FSGID:
                        result = audit_gid_comparator(cred->fsgid, f->op, f->gid);
                        break;
                case AUDIT_SESSIONID:
                        sessionid = audit_get_sessionid(tsk);
                        result = audit_comparator(sessionid, f->op, f->val);
                        break;
                case AUDIT_PERS:
                        result = audit_comparator(tsk->personality, f->op, f->val);
                        break;
                case AUDIT_ARCH:
                        if (ctx)
                                result = audit_comparator(ctx->arch, f->op, f->val);
                        break;

                case AUDIT_EXIT:
                        if (ctx && ctx->return_valid != AUDITSC_INVALID)
                                result = audit_comparator(ctx->return_code, f->op, f->val);
                        break;
                case AUDIT_SUCCESS:
                        if (ctx && ctx->return_valid != AUDITSC_INVALID) {
                                if (f->val)
                                        result = audit_comparator(ctx->return_valid, f->op, AUDITSC_SUCCESS);
                                else
                                        result = audit_comparator(ctx->return_valid, f->op, AUDITSC_FAILURE);
                        }
                        break;
                case AUDIT_DEVMAJOR:
                        if (name) {
                                if (audit_comparator(MAJOR(name->dev), f->op, f->val) ||
                                    audit_comparator(MAJOR(name->rdev), f->op, f->val))
                                        ++result;
                        } else if (ctx) {
                                list_for_each_entry(n, &ctx->names_list, list) {
                                        if (audit_comparator(MAJOR(n->dev), f->op, f->val) ||
                                            audit_comparator(MAJOR(n->rdev), f->op, f->val)) {
                                                ++result;
                                                break;
                                        }
                                }
                        }
                        break;
                case AUDIT_DEVMINOR:
                        if (name) {
                                if (audit_comparator(MINOR(name->dev), f->op, f->val) ||
                                    audit_comparator(MINOR(name->rdev), f->op, f->val))
                                        ++result;
                        } else if (ctx) {
                                list_for_each_entry(n, &ctx->names_list, list) {
                                        if (audit_comparator(MINOR(n->dev), f->op, f->val) ||
                                            audit_comparator(MINOR(n->rdev), f->op, f->val)) {
                                                ++result;
                                                break;
                                        }
                                }
                        }
                        break;
                case AUDIT_INODE:
                        if (name)
                                result = audit_comparator(name->ino, f->op, f->val);
                        else if (ctx) {
                                list_for_each_entry(n, &ctx->names_list, list) {
                                        if (audit_comparator(n->ino, f->op, f->val)) {
                                                ++result;
                                                break;
                                        }
                                }
                        }
                        break;
                case AUDIT_OBJ_UID:
                        if (name) {
                                result = audit_uid_comparator(name->uid, f->op, f->uid);
                        } else if (ctx) {
                                list_for_each_entry(n, &ctx->names_list, list) {
                                        if (audit_uid_comparator(n->uid, f->op, f->uid)) {
                                                ++result;
                                                break;
                                        }
                                }
                        }
                        break;
                case AUDIT_OBJ_GID:
                        if (name) {
                                result = audit_gid_comparator(name->gid, f->op, f->gid);
                        } else if (ctx) {
                                list_for_each_entry(n, &ctx->names_list, list) {
                                        if (audit_gid_comparator(n->gid, f->op, f->gid)) {
                                                ++result;
                                                break;
                                        }
                                }
                        }
                        break;
                case AUDIT_WATCH:
                        if (name) {
                                result = audit_watch_compare(rule->watch,
                                                             name->ino,
                                                             name->dev);
                                if (f->op == Audit_not_equal)
                                        result = !result;
                        }
                        break;
                case AUDIT_DIR:
                        if (ctx) {
                                result = match_tree_refs(ctx, rule->tree);
                                if (f->op == Audit_not_equal)
                                        result = !result;
                        }
                        break;
                case AUDIT_LOGINUID:
                        result = audit_uid_comparator(audit_get_loginuid(tsk),
                                                      f->op, f->uid);
                        break;
                case AUDIT_LOGINUID_SET:
                        result = audit_comparator(audit_loginuid_set(tsk), f->op, f->val);
                        break;
                case AUDIT_SADDR_FAM:
                        if (ctx && ctx->sockaddr)
                                result = audit_comparator(ctx->sockaddr->ss_family,
                                                          f->op, f->val);
                        break;
                case AUDIT_SUBJ_USER:
                case AUDIT_SUBJ_ROLE:
                case AUDIT_SUBJ_TYPE:
                case AUDIT_SUBJ_SEN:
                case AUDIT_SUBJ_CLR:
                        /* NOTE: this may return negative values indicating
                           a temporary error.  We simply treat this as a
                           match for now to avoid losing information that
                           may be wanted.   An error message will also be
                           logged upon error */
                        if (f->lsm_rule) {
                                if (need_sid) {
                                        /* @tsk should always be equal to
                                         * @current with the exception of
                                         * fork()/copy_process() in which case
                                         * the new @tsk creds are still a dup
                                         * of @current's creds so we can still
                                         * use
                                         * security_current_getlsmprop_subj()
                                         * here even though it always refs
                                         * @current's creds
                                         */
                                        security_current_getlsmprop_subj(&prop);
                                        need_sid = 0;
                                }
                                result = security_audit_rule_match(&prop,
                                                                   f->type,
                                                                   f->op,
                                                                   f->lsm_rule);
                        }
                        break;
                case AUDIT_OBJ_USER:
                case AUDIT_OBJ_ROLE:
                case AUDIT_OBJ_TYPE:
                case AUDIT_OBJ_LEV_LOW:
                case AUDIT_OBJ_LEV_HIGH:
                        /* The above note for AUDIT_SUBJ_USER...AUDIT_SUBJ_CLR
                           also applies here */
                        if (f->lsm_rule) {
                                /* Find files that match */
                                if (name) {
                                        result = security_audit_rule_match(
                                                                &name->oprop,
                                                                f->type,
                                                                f->op,
                                                                f->lsm_rule);
                                } else if (ctx) {
                                        list_for_each_entry(n, &ctx->names_list, list) {
                                                if (security_audit_rule_match(
                                                                &n->oprop,
                                                                f->type,
                                                                f->op,
                                                                f->lsm_rule)) {
                                                        ++result;
                                                        break;
                                                }
                                        }
                                }
                                /* Find ipc objects that match */
                                if (!ctx || ctx->type != AUDIT_IPC)
                                        break;
                                if (security_audit_rule_match(&ctx->ipc.oprop,
                                                              f->type, f->op,
                                                              f->lsm_rule))
                                        ++result;
                        }
                        break;
                case AUDIT_ARG0:
                case AUDIT_ARG1:
                case AUDIT_ARG2:
                case AUDIT_ARG3:
                        if (ctx)
                                result = audit_comparator(ctx->argv[f->type-AUDIT_ARG0], f->op, f->val);
                        break;
                case AUDIT_FILTERKEY:
                        /* ignore this field for filtering */
                        result = 1;
                        break;
                case AUDIT_PERM:
                        result = audit_match_perm(ctx, f->val);
                        if (f->op == Audit_not_equal)
                                result = !result;
                        break;
                case AUDIT_FILETYPE:
                        result = audit_match_filetype(ctx, f->val);
                        if (f->op == Audit_not_equal)
                                result = !result;
                        break;
                case AUDIT_FIELD_COMPARE:
                        result = audit_field_compare(tsk, cred, f, ctx, name);
                        break;
                }
                if (!result)
                        return 0;
        }

        if (ctx) {
                if (rule->filterkey) {
                        kfree(ctx->filterkey);
                        ctx->filterkey = kstrdup(rule->filterkey, GFP_ATOMIC);
                }
                ctx->prio = rule->prio;
        }
        switch (rule->action) {
        case AUDIT_NEVER:
                *state = AUDIT_STATE_DISABLED;
                break;
        case AUDIT_ALWAYS:
                *state = AUDIT_STATE_RECORD;
                break;
        }
        return 1;
}

/* At process creation time, we can determine if system-call auditing is
 * completely disabled for this task.  Since we only have the task
 * structure at this point, we can only check uid and gid.
 */
static enum audit_state audit_filter_task(struct task_struct *tsk, char **key)
{
        struct audit_entry *e;
        enum audit_state   state;

        rcu_read_lock();
        list_for_each_entry_rcu(e, &audit_filter_list[AUDIT_FILTER_TASK], list) {
                if (audit_filter_rules(tsk, &e->rule, NULL, NULL,
                                       &state, true)) {
                        if (state == AUDIT_STATE_RECORD)
                                *key = kstrdup(e->rule.filterkey, GFP_ATOMIC);
                        rcu_read_unlock();
                        return state;
                }
        }
        rcu_read_unlock();
        return AUDIT_STATE_BUILD;
}

static int audit_in_mask(const struct audit_krule *rule, unsigned long val)
{
        int word, bit;

        if (val > 0xffffffff)
                return false;

        word = AUDIT_WORD(val);
        if (word >= AUDIT_BITMASK_SIZE)
                return false;

        bit = AUDIT_BIT(val);

        return rule->mask[word] & bit;
}

/**
 * __audit_filter_op - common filter helper for operations (syscall/uring/etc)
 * @tsk: associated task
 * @ctx: audit context
 * @list: audit filter list
 * @name: audit_name (can be NULL)
 * @op: current syscall/uring_op
 *
 * Run the udit filters specified in @list against @tsk using @ctx,
 * @name, and @op, as necessary; the caller is responsible for ensuring
 * that the call is made while the RCU read lock is held. The @name
 * parameter can be NULL, but all others must be specified.
 * Returns 1/true if the filter finds a match, 0/false if none are found.
 */
static int __audit_filter_op(struct task_struct *tsk,
                           struct audit_context *ctx,
                           struct list_head *list,
                           struct audit_names *name,
                           unsigned long op)
{
        struct audit_entry *e;
        enum audit_state state;

        list_for_each_entry_rcu(e, list, list) {
                if (audit_in_mask(&e->rule, op) &&
                    audit_filter_rules(tsk, &e->rule, ctx, name,
                                       &state, false)) {
                        ctx->current_state = state;
                        return 1;
                }
        }
        return 0;
}

/**
 * audit_filter_uring - apply filters to an io_uring operation
 * @tsk: associated task
 * @ctx: audit context
 */
static void audit_filter_uring(struct task_struct *tsk,
                               struct audit_context *ctx)
{
        if (auditd_test_task(tsk))
                return;

        rcu_read_lock();
        __audit_filter_op(tsk, ctx, &audit_filter_list[AUDIT_FILTER_URING_EXIT],
                        NULL, ctx->uring_op);
        rcu_read_unlock();
}

/* At syscall exit time, this filter is called if the audit_state is
 * not low enough that auditing cannot take place, but is also not
 * high enough that we already know we have to write an audit record
 * (i.e., the state is AUDIT_STATE_BUILD).
 */
static void audit_filter_syscall(struct task_struct *tsk,
                                 struct audit_context *ctx)
{
        if (auditd_test_task(tsk))
                return;

        rcu_read_lock();
        __audit_filter_op(tsk, ctx, &audit_filter_list[AUDIT_FILTER_EXIT],
                        NULL, ctx->major);
        rcu_read_unlock();
}

/*
 * Given an audit_name check the inode hash table to see if they match.
 * Called holding the rcu read lock to protect the use of audit_inode_hash
 */
static int audit_filter_inode_name(struct task_struct *tsk,
                                   struct audit_names *n,
                                   struct audit_context *ctx)
{
        int h = audit_hash_ino(n->ino);
        struct list_head *list = &audit_inode_hash[h];

        return __audit_filter_op(tsk, ctx, list, n, ctx->major);
}

/* At syscall exit time, this filter is called if any audit_names have been
 * collected during syscall processing.  We only check rules in sublists at hash
 * buckets applicable to the inode numbers in audit_names.
 * Regarding audit_state, same rules apply as for audit_filter_syscall().
 */
void audit_filter_inodes(struct task_struct *tsk, struct audit_context *ctx)
{
        struct audit_names *n;

        if (auditd_test_task(tsk))
                return;

        rcu_read_lock();

        list_for_each_entry(n, &ctx->names_list, list) {
                if (audit_filter_inode_name(tsk, n, ctx))
                        break;
        }
        rcu_read_unlock();
}

static inline void audit_proctitle_free(struct audit_context *context)
{
        kfree(context->proctitle.value);
        context->proctitle.value = NULL;
        context->proctitle.len = 0;
}

static inline void audit_free_module(struct audit_context *context)
{
        if (context->type == AUDIT_KERN_MODULE) {
                kfree(context->module.name);
                context->module.name = NULL;
        }
}
static inline void audit_free_names(struct audit_context *context)
{
        struct audit_names *n, *next;

        list_for_each_entry_safe(n, next, &context->names_list, list) {
                list_del(&n->list);
                if (n->name)
                        putname(n->name);
                if (n->should_free)
                        kfree(n);
        }
        context->name_count = 0;
        path_put(&context->pwd);
        context->pwd.dentry = NULL;
        context->pwd.mnt = NULL;
}

static inline void audit_free_aux(struct audit_context *context)
{
        struct audit_aux_data *aux;

        while ((aux = context->aux)) {
                context->aux = aux->next;
                kfree(aux);
        }
        context->aux = NULL;
        while ((aux = context->aux_pids)) {
                context->aux_pids = aux->next;
                kfree(aux);
        }
        context->aux_pids = NULL;
}

/**
 * audit_reset_context - reset a audit_context structure
 * @ctx: the audit_context to reset
 *
 * All fields in the audit_context will be reset to an initial state, all
 * references held by fields will be dropped, and private memory will be
 * released.  When this function returns the audit_context will be suitable
 * for reuse, so long as the passed context is not NULL or a dummy context.
 */
static void audit_reset_context(struct audit_context *ctx)
{
        if (!ctx)
                return;

        /* if ctx is non-null, reset the "ctx->context" regardless */
        ctx->context = AUDIT_CTX_UNUSED;
        if (ctx->dummy)
                return;

        /*
         * NOTE: It shouldn't matter in what order we release the fields, so
         *       release them in the order in which they appear in the struct;
         *       this gives us some hope of quickly making sure we are
         *       resetting the audit_context properly.
         *
         *       Other things worth mentioning:
         *       - we don't reset "dummy"
         *       - we don't reset "state", we do reset "current_state"
         *       - we preserve "filterkey" if "state" is AUDIT_STATE_RECORD
         *       - much of this is likely overkill, but play it safe for now
         *       - we really need to work on improving the audit_context struct
         */

        ctx->current_state = ctx->state;
        ctx->stamp.serial = 0;
        ctx->stamp.ctime = (struct timespec64){ .tv_sec = 0, .tv_nsec = 0 };
        ctx->major = 0;
        ctx->uring_op = 0;
        memset(ctx->argv, 0, sizeof(ctx->argv));
        ctx->return_code = 0;
        ctx->prio = (ctx->state == AUDIT_STATE_RECORD ? ~0ULL : 0);
        ctx->return_valid = AUDITSC_INVALID;
        audit_free_names(ctx);
        if (ctx->state != AUDIT_STATE_RECORD) {
                kfree(ctx->filterkey);
                ctx->filterkey = NULL;
        }
        audit_free_aux(ctx);
        kfree(ctx->sockaddr);
        ctx->sockaddr = NULL;
        ctx->sockaddr_len = 0;
        ctx->ppid = 0;
        ctx->uid = ctx->euid = ctx->suid = ctx->fsuid = KUIDT_INIT(0);
        ctx->gid = ctx->egid = ctx->sgid = ctx->fsgid = KGIDT_INIT(0);
        ctx->personality = 0;
        ctx->arch = 0;
        ctx->target_pid = 0;
        ctx->target_auid = ctx->target_uid = KUIDT_INIT(0);
        ctx->target_sessionid = 0;
        lsmprop_init(&ctx->target_ref);
        ctx->target_comm[0] = '\0';
        unroll_tree_refs(ctx, NULL, 0);
        WARN_ON(!list_empty(&ctx->killed_trees));
        audit_free_module(ctx);
        ctx->fds[0] = -1;
        ctx->type = 0; /* reset last for audit_free_*() */
}

static inline struct audit_context *audit_alloc_context(enum audit_state state)
{
        struct audit_context *context;

        context = kzalloc_obj(*context);
        if (!context)
                return NULL;
        context->context = AUDIT_CTX_UNUSED;
        context->state = state;
        context->prio = state == AUDIT_STATE_RECORD ? ~0ULL : 0;
        INIT_LIST_HEAD(&context->killed_trees);
        INIT_LIST_HEAD(&context->names_list);
        context->fds[0] = -1;
        context->return_valid = AUDITSC_INVALID;
        return context;
}

/**
 * audit_alloc - allocate an audit context block for a task
 * @tsk: task
 *
 * Filter on the task information and allocate a per-task audit context
 * if necessary.  Doing so turns on system call auditing for the
 * specified task.  This is called from copy_process, so no lock is
 * needed.
 */
int audit_alloc(struct task_struct *tsk)
{
        struct audit_context *context;
        enum audit_state     state;
        char *key = NULL;

        if (likely(!audit_ever_enabled))
                return 0;

        state = audit_filter_task(tsk, &key);
        if (state == AUDIT_STATE_DISABLED) {
                clear_task_syscall_work(tsk, SYSCALL_AUDIT);
                return 0;
        }

        context = audit_alloc_context(state);
        if (!context) {
                kfree(key);
                audit_log_lost("out of memory in audit_alloc");
                return -ENOMEM;
        }
        context->filterkey = key;

        audit_set_context(tsk, context);
        set_task_syscall_work(tsk, SYSCALL_AUDIT);
        return 0;
}

static inline void audit_free_context(struct audit_context *context)
{
        /* resetting is extra work, but it is likely just noise */
        audit_reset_context(context);
        audit_proctitle_free(context);
        free_tree_refs(context);
        kfree(context->filterkey);
        kfree(context);
}

static int audit_log_pid_context(struct audit_context *context, pid_t pid,
                                 kuid_t auid, kuid_t uid,
                                 unsigned int sessionid, struct lsm_prop *prop,
                                 char *comm)
{
        struct audit_buffer *ab;
        int rc = 0;

        ab = audit_log_start(context, GFP_KERNEL, AUDIT_OBJ_PID);
        if (!ab)
                return rc;

        audit_log_format(ab, "opid=%d oauid=%d ouid=%d oses=%d", pid,
                         from_kuid(&init_user_ns, auid),
                         from_kuid(&init_user_ns, uid), sessionid);
        if (lsmprop_is_set(prop) && audit_log_obj_ctx(ab, prop))
                rc = 1;

        audit_log_format(ab, " ocomm=");
        audit_log_untrustedstring(ab, comm);
        audit_log_end(ab);

        return rc;
}

static void audit_log_execve_info(struct audit_context *context,
                                  struct audit_buffer **ab)
{
        long len_max;
        long len_rem;
        long len_full;
        long len_buf;
        long len_abuf = 0;
        long len_tmp;
        bool require_data;
        bool encode;
        unsigned int iter;
        unsigned int arg;
        char *buf_head;
        char *buf;
        const char __user *p = (const char __user *)current->mm->arg_start;

        /* NOTE: this buffer needs to be large enough to hold all the non-arg
         *       data we put in the audit record for this argument (see the
         *       code below) ... at this point in time 96 is plenty */
        char abuf[96];

        /* NOTE: we set MAX_EXECVE_AUDIT_LEN to a rather arbitrary limit, the
         *       current value of 7500 is not as important as the fact that it
         *       is less than 8k, a setting of 7500 gives us plenty of wiggle
         *       room if we go over a little bit in the logging below */
        WARN_ON_ONCE(MAX_EXECVE_AUDIT_LEN > 7500);
        len_max = MAX_EXECVE_AUDIT_LEN;

        /* scratch buffer to hold the userspace args */
        buf_head = kmalloc(MAX_EXECVE_AUDIT_LEN + 1, GFP_KERNEL);
        if (!buf_head) {
                audit_panic("out of memory for argv string");
                return;
        }
        buf = buf_head;

        audit_log_format(*ab, "argc=%d", context->execve.argc);

        len_rem = len_max;
        len_buf = 0;
        len_full = 0;
        require_data = true;
        encode = false;
        iter = 0;
        arg = 0;
        do {
                /* NOTE: we don't ever want to trust this value for anything
                 *       serious, but the audit record format insists we
                 *       provide an argument length for really long arguments,
                 *       e.g. > MAX_EXECVE_AUDIT_LEN, so we have no choice but
                 *       to use strncpy_from_user() to obtain this value for
                 *       recording in the log, although we don't use it
                 *       anywhere here to avoid a double-fetch problem */
                if (len_full == 0)
                        len_full = strnlen_user(p, MAX_ARG_STRLEN) - 1;

                /* read more data from userspace */
                if (require_data) {
                        /* can we make more room in the buffer? */
                        if (buf != buf_head) {
                                memmove(buf_head, buf, len_buf);
                                buf = buf_head;
                        }

                        /* fetch as much as we can of the argument */
                        len_tmp = strncpy_from_user(&buf_head[len_buf], p,
                                                    len_max - len_buf);
                        if (len_tmp == -EFAULT) {
                                /* unable to copy from userspace */
                                send_sig(SIGKILL, current, 0);
                                goto out;
                        } else if (len_tmp == (len_max - len_buf)) {
                                /* buffer is not large enough */
                                require_data = true;
                                /* NOTE: if we are going to span multiple
                                 *       buffers force the encoding so we stand
                                 *       a chance at a sane len_full value and
                                 *       consistent record encoding */
                                encode = true;
                                len_full = len_full * 2;
                                p += len_tmp;
                        } else {
                                require_data = false;
                                if (!encode)
                                        encode = audit_string_contains_control(
                                                                buf, len_tmp);
                                /* try to use a trusted value for len_full */
                                if (len_full < len_max)
                                        len_full = (encode ?
                                                    len_tmp * 2 : len_tmp);
                                p += len_tmp + 1;
                        }
                        len_buf += len_tmp;
                        buf_head[len_buf] = '\0';

                        /* length of the buffer in the audit record? */
                        len_abuf = (encode ? len_buf * 2 : len_buf + 2);
                }

                /* write as much as we can to the audit log */
                if (len_buf >= 0) {
                        /* NOTE: some magic numbers here - basically if we
                         *       can't fit a reasonable amount of data into the
                         *       existing audit buffer, flush it and start with
                         *       a new buffer */
                        if ((sizeof(abuf) + 8) > len_rem) {
                                len_rem = len_max;
                                audit_log_end(*ab);
                                *ab = audit_log_start(context,
                                                      GFP_KERNEL, AUDIT_EXECVE);
                                if (!*ab)
                                        goto out;
                        }

                        /* create the non-arg portion of the arg record */
                        len_tmp = 0;
                        if (require_data || (iter > 0) ||
                            ((len_abuf + sizeof(abuf)) > len_rem)) {
                                if (iter == 0) {
                                        len_tmp += snprintf(&abuf[len_tmp],
                                                        sizeof(abuf) - len_tmp,
                                                        " a%d_len=%lu",
                                                        arg, len_full);
                                }
                                len_tmp += snprintf(&abuf[len_tmp],
                                                    sizeof(abuf) - len_tmp,
                                                    " a%d[%d]=", arg, iter++);
                        } else
                                len_tmp += snprintf(&abuf[len_tmp],
                                                    sizeof(abuf) - len_tmp,
                                                    " a%d=", arg);
                        WARN_ON(len_tmp >= sizeof(abuf));
                        abuf[sizeof(abuf) - 1] = '\0';

                        /* log the arg in the audit record */
                        audit_log_format(*ab, "%s", abuf);
                        len_rem -= len_tmp;
                        len_tmp = len_buf;
                        if (encode) {
                                if (len_abuf > len_rem)
                                        len_tmp = len_rem / 2; /* encoding */
                                audit_log_n_hex(*ab, buf, len_tmp);
                                len_rem -= len_tmp * 2;
                                len_abuf -= len_tmp * 2;
                        } else {
                                if (len_abuf > len_rem)
                                        len_tmp = len_rem - 2; /* quotes */
                                audit_log_n_string(*ab, buf, len_tmp);
                                len_rem -= len_tmp + 2;
                                /* don't subtract the "2" because we still need
                                 * to add quotes to the remaining string */
                                len_abuf -= len_tmp;
                        }
                        len_buf -= len_tmp;
                        buf += len_tmp;
                }

                /* ready to move to the next argument? */
                if ((len_buf == 0) && !require_data) {
                        arg++;
                        iter = 0;
                        len_full = 0;
                        require_data = true;
                        encode = false;
                }
        } while (arg < context->execve.argc);

        /* NOTE: the caller handles the final audit_log_end() call */

out:
        kfree(buf_head);
}

static void audit_log_cap(struct audit_buffer *ab, char *prefix,
                          kernel_cap_t *cap)
{
        if (cap_isclear(*cap)) {
                audit_log_format(ab, " %s=0", prefix);
                return;
        }
        audit_log_format(ab, " %s=%016llx", prefix, cap->val);
}

static void audit_log_fcaps(struct audit_buffer *ab, struct audit_names *name)
{
        if (name->fcap_ver == -1) {
                audit_log_format(ab, " cap_fe=? cap_fver=? cap_fp=? cap_fi=?");
                return;
        }
        audit_log_cap(ab, "cap_fp", &name->fcap.permitted);
        audit_log_cap(ab, "cap_fi", &name->fcap.inheritable);
        audit_log_format(ab, " cap_fe=%d cap_fver=%x cap_frootid=%d",
                         name->fcap.fE, name->fcap_ver,
                         from_kuid(&init_user_ns, name->fcap.rootid));
}

static void audit_log_time(struct audit_context *context, struct audit_buffer **ab)
{
        const struct audit_ntp_data *ntp = &context->time.ntp_data;
        const struct timespec64 *tk = &context->time.tk_injoffset;
        static const char * const ntp_name[] = {
                "offset",
                "freq",
                "status",
                "tai",
                "tick",
                "adjust",
        };
        int type;

        if (context->type == AUDIT_TIME_ADJNTPVAL) {
                for (type = 0; type < AUDIT_NTP_NVALS; type++) {
                        if (ntp->vals[type].newval != ntp->vals[type].oldval) {
                                if (!*ab) {
                                        *ab = audit_log_start(context,
                                                        GFP_KERNEL,
                                                        AUDIT_TIME_ADJNTPVAL);
                                        if (!*ab)
                                                return;
                                }
                                audit_log_format(*ab, "op=%s old=%lli new=%lli",
                                                 ntp_name[type],
                                                 ntp->vals[type].oldval,
                                                 ntp->vals[type].newval);
                                audit_log_end(*ab);
                                *ab = NULL;
                        }
                }
        }
        if (tk->tv_sec != 0 || tk->tv_nsec != 0) {
                if (!*ab) {
                        *ab = audit_log_start(context, GFP_KERNEL,
                                              AUDIT_TIME_INJOFFSET);
                        if (!*ab)
                                return;
                }
                audit_log_format(*ab, "sec=%lli nsec=%li",
                                 (long long)tk->tv_sec, tk->tv_nsec);
                audit_log_end(*ab);
                *ab = NULL;
        }
}

static void show_special(struct audit_context *context, int *call_panic)
{
        struct audit_buffer *ab;
        int i;

        ab = audit_log_start(context, GFP_KERNEL, context->type);
        if (!ab)
                return;

        switch (context->type) {
        case AUDIT_SOCKETCALL: {
                int nargs = context->socketcall.nargs;

                audit_log_format(ab, "nargs=%d", nargs);
                for (i = 0; i < nargs; i++)
                        audit_log_format(ab, " a%d=%lx", i,
                                context->socketcall.args[i]);
                break; }
        case AUDIT_IPC:
                audit_log_format(ab, "ouid=%u ogid=%u mode=%#ho",
                                 from_kuid(&init_user_ns, context->ipc.uid),
                                 from_kgid(&init_user_ns, context->ipc.gid),
                                 context->ipc.mode);
                if (lsmprop_is_set(&context->ipc.oprop)) {
                        if (audit_log_obj_ctx(ab, &context->ipc.oprop))
                                *call_panic = 1;
                }
                if (context->ipc.has_perm) {
                        audit_log_end(ab);
                        ab = audit_log_start(context, GFP_KERNEL,
                                             AUDIT_IPC_SET_PERM);
                        if (unlikely(!ab))
                                return;
                        audit_log_format(ab,
                                "qbytes=%lx ouid=%u ogid=%u mode=%#ho",
                                context->ipc.qbytes,
                                context->ipc.perm_uid,
                                context->ipc.perm_gid,
                                context->ipc.perm_mode);
                }
                break;
        case AUDIT_MQ_OPEN:
                audit_log_format(ab,
                        "oflag=0x%x mode=%#ho mq_flags=0x%lx mq_maxmsg=%ld "
                        "mq_msgsize=%ld mq_curmsgs=%ld",
                        context->mq_open.oflag, context->mq_open.mode,
                        context->mq_open.attr.mq_flags,
                        context->mq_open.attr.mq_maxmsg,
                        context->mq_open.attr.mq_msgsize,
                        context->mq_open.attr.mq_curmsgs);
                break;
        case AUDIT_MQ_SENDRECV:
                audit_log_format(ab,
                        "mqdes=%d msg_len=%zd msg_prio=%u "
                        "abs_timeout_sec=%lld abs_timeout_nsec=%ld",
                        context->mq_sendrecv.mqdes,
                        context->mq_sendrecv.msg_len,
                        context->mq_sendrecv.msg_prio,
                        (long long) context->mq_sendrecv.abs_timeout.tv_sec,
                        context->mq_sendrecv.abs_timeout.tv_nsec);
                break;
        case AUDIT_MQ_NOTIFY:
                audit_log_format(ab, "mqdes=%d sigev_signo=%d",
                                context->mq_notify.mqdes,
                                context->mq_notify.sigev_signo);
                break;
        case AUDIT_MQ_GETSETATTR: {
                struct mq_attr *attr = &context->mq_getsetattr.mqstat;

                audit_log_format(ab,
                        "mqdes=%d mq_flags=0x%lx mq_maxmsg=%ld mq_msgsize=%ld "
                        "mq_curmsgs=%ld ",
                        context->mq_getsetattr.mqdes,
                        attr->mq_flags, attr->mq_maxmsg,
                        attr->mq_msgsize, attr->mq_curmsgs);
                break; }
        case AUDIT_CAPSET:
                audit_log_format(ab, "pid=%d", context->capset.pid);
                audit_log_cap(ab, "cap_pi", &context->capset.cap.inheritable);
                audit_log_cap(ab, "cap_pp", &context->capset.cap.permitted);
                audit_log_cap(ab, "cap_pe", &context->capset.cap.effective);
                audit_log_cap(ab, "cap_pa", &context->capset.cap.ambient);
                break;
        case AUDIT_MMAP:
                audit_log_format(ab, "fd=%d flags=0x%x", context->mmap.fd,
                                 context->mmap.flags);
                break;
        case AUDIT_OPENAT2:
                audit_log_format(ab, "oflag=0%llo mode=0%llo resolve=0x%llx",
                                 context->openat2.flags,
                                 context->openat2.mode,
                                 context->openat2.resolve);
                break;
        case AUDIT_EXECVE:
                audit_log_execve_info(context, &ab);
                break;
        case AUDIT_KERN_MODULE:
                audit_log_format(ab, "name=");
                if (context->module.name) {
                        audit_log_untrustedstring(ab, context->module.name);
                } else
                        audit_log_format(ab, "(null)");

                break;
        case AUDIT_TIME_ADJNTPVAL:
        case AUDIT_TIME_INJOFFSET:
                /* this call deviates from the rest, eating the buffer */
                audit_log_time(context, &ab);
                break;
        }
        audit_log_end(ab);
}

static inline int audit_proctitle_rtrim(char *proctitle, int len)
{
        char *end = proctitle + len - 1;

        while (end > proctitle && !isprint(*end))
                end--;

        /* catch the case where proctitle is only 1 non-print character */
        len = end - proctitle + 1;
        len -= isprint(proctitle[len-1]) == 0;
        return len;
}

/*
 * audit_log_name - produce AUDIT_PATH record from struct audit_names
 * @context: audit_context for the task
 * @n: audit_names structure with reportable details
 * @path: optional path to report instead of audit_names->name
 * @record_num: record number to report when handling a list of names
 * @call_panic: optional pointer to int that will be updated if secid fails
 */
static void audit_log_name(struct audit_context *context, struct audit_names *n,
                    const struct path *path, int record_num, int *call_panic)
{
        struct audit_buffer *ab;

        ab = audit_log_start(context, GFP_KERNEL, AUDIT_PATH);
        if (!ab)
                return;

        audit_log_format(ab, "item=%d", record_num);

        if (path)
                audit_log_d_path(ab, " name=", path);
        else if (n->name) {
                switch (n->name_len) {
                case AUDIT_NAME_FULL:
                        /* log the full path */
                        audit_log_format(ab, " name=");
                        audit_log_untrustedstring(ab, n->name->name);
                        break;
                case 0:
                        /* name was specified as a relative path and the
                         * directory component is the cwd
                         */
                        if (context->pwd.dentry && context->pwd.mnt)
                                audit_log_d_path(ab, " name=", &context->pwd);
                        else
                                audit_log_format(ab, " name=(null)");
                        break;
                default:
                        /* log the name's directory component */
                        audit_log_format(ab, " name=");
                        audit_log_n_untrustedstring(ab, n->name->name,
                                                    n->name_len);
                }
        } else
                audit_log_format(ab, " name=(null)");

        if (n->ino != AUDIT_INO_UNSET)
                audit_log_format(ab, " inode=%llu dev=%02x:%02x mode=%#ho ouid=%u ogid=%u rdev=%02x:%02x",
                                 n->ino,
                                 MAJOR(n->dev),
                                 MINOR(n->dev),
                                 n->mode,
                                 from_kuid(&init_user_ns, n->uid),
                                 from_kgid(&init_user_ns, n->gid),
                                 MAJOR(n->rdev),
                                 MINOR(n->rdev));
        if (lsmprop_is_set(&n->oprop) &&
            audit_log_obj_ctx(ab, &n->oprop))
                *call_panic = 2;

        /* log the audit_names record type */
        switch (n->type) {
        case AUDIT_TYPE_NORMAL:
                audit_log_format(ab, " nametype=NORMAL");
                break;
        case AUDIT_TYPE_PARENT:
                audit_log_format(ab, " nametype=PARENT");
                break;
        case AUDIT_TYPE_CHILD_DELETE:
                audit_log_format(ab, " nametype=DELETE");
                break;
        case AUDIT_TYPE_CHILD_CREATE:
                audit_log_format(ab, " nametype=CREATE");
                break;
        default:
                audit_log_format(ab, " nametype=UNKNOWN");
                break;
        }

        audit_log_fcaps(ab, n);
        audit_log_end(ab);
}

static void audit_log_proctitle(void)
{
        int res;
        char *buf;
        char *msg = "(null)";
        int len = strlen(msg);
        struct audit_context *context = audit_context();
        struct audit_buffer *ab;

        ab = audit_log_start(context, GFP_KERNEL, AUDIT_PROCTITLE);
        if (!ab)
                return;        /* audit_panic or being filtered */

        audit_log_format(ab, "proctitle=");

        /* Not  cached */
        if (!context->proctitle.value) {
                buf = kmalloc(MAX_PROCTITLE_AUDIT_LEN, GFP_KERNEL);
                if (!buf)
                        goto out;
                /* Historically called this from procfs naming */
                res = get_cmdline(current, buf, MAX_PROCTITLE_AUDIT_LEN);
                if (res == 0) {
                        kfree(buf);
                        goto out;
                }
                res = audit_proctitle_rtrim(buf, res);
                if (res == 0) {
                        kfree(buf);
                        goto out;
                }
                context->proctitle.value = buf;
                context->proctitle.len = res;
        }
        msg = context->proctitle.value;
        len = context->proctitle.len;
out:
        audit_log_n_untrustedstring(ab, msg, len);
        audit_log_end(ab);
}

/**
 * audit_log_uring - generate a AUDIT_URINGOP record
 * @ctx: the audit context
 */
static void audit_log_uring(struct audit_context *ctx)
{
        struct audit_buffer *ab;
        const struct cred *cred;

        ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_URINGOP);
        if (!ab)
                return;
        cred = current_cred();
        audit_log_format(ab, "uring_op=%d", ctx->uring_op);
        if (ctx->return_valid != AUDITSC_INVALID)
                audit_log_format(ab, " success=%s exit=%ld",
                                 str_yes_no(ctx->return_valid ==
                                            AUDITSC_SUCCESS),
                                 ctx->return_code);
        audit_log_format(ab,
                         " items=%d"
                         " ppid=%d pid=%d uid=%u gid=%u euid=%u suid=%u"
                         " fsuid=%u egid=%u sgid=%u fsgid=%u",
                         ctx->name_count,
                         task_ppid_nr(current), task_tgid_nr(current),
                         from_kuid(&init_user_ns, cred->uid),
                         from_kgid(&init_user_ns, cred->gid),
                         from_kuid(&init_user_ns, cred->euid),
                         from_kuid(&init_user_ns, cred->suid),
                         from_kuid(&init_user_ns, cred->fsuid),
                         from_kgid(&init_user_ns, cred->egid),
                         from_kgid(&init_user_ns, cred->sgid),
                         from_kgid(&init_user_ns, cred->fsgid));
        audit_log_task_context(ab);
        audit_log_key(ab, ctx->filterkey);
        audit_log_end(ab);
}

static void audit_log_exit(void)
{
        int i, call_panic = 0;
        struct audit_context *context = audit_context();
        struct audit_buffer *ab;
        struct audit_aux_data *aux;
        struct audit_names *n;

        context->personality = current->personality;

        switch (context->context) {
        case AUDIT_CTX_SYSCALL:
                ab = audit_log_start(context, GFP_KERNEL, AUDIT_SYSCALL);
                if (!ab)
                        return;
                audit_log_format(ab, "arch=%x syscall=%d",
                                 context->arch, context->major);
                if (context->personality != PER_LINUX)
                        audit_log_format(ab, " per=%lx", context->personality);
                if (context->return_valid != AUDITSC_INVALID)
                        audit_log_format(ab, " success=%s exit=%ld",
                                         str_yes_no(context->return_valid ==
                                                    AUDITSC_SUCCESS),
                                         context->return_code);
                audit_log_format(ab,
                                 " a0=%lx a1=%lx a2=%lx a3=%lx items=%d",
                                 context->argv[0],
                                 context->argv[1],
                                 context->argv[2],
                                 context->argv[3],
                                 context->name_count);
                audit_log_task_info(ab);
                audit_log_key(ab, context->filterkey);
                audit_log_end(ab);
                break;
        case AUDIT_CTX_URING:
                audit_log_uring(context);
                break;
        default:
                BUG();
                break;
        }

        for (aux = context->aux; aux; aux = aux->next) {

                ab = audit_log_start(context, GFP_KERNEL, aux->type);
                if (!ab)
                        continue; /* audit_panic has been called */

                switch (aux->type) {

                case AUDIT_BPRM_FCAPS: {
                        struct audit_aux_data_bprm_fcaps *axs = (void *)aux;

                        audit_log_format(ab, "fver=%x", axs->fcap_ver);
                        audit_log_cap(ab, "fp", &axs->fcap.permitted);
                        audit_log_cap(ab, "fi", &axs->fcap.inheritable);
                        audit_log_format(ab, " fe=%d", axs->fcap.fE);
                        audit_log_cap(ab, "old_pp", &axs->old_pcap.permitted);
                        audit_log_cap(ab, "old_pi", &axs->old_pcap.inheritable);
                        audit_log_cap(ab, "old_pe", &axs->old_pcap.effective);
                        audit_log_cap(ab, "old_pa", &axs->old_pcap.ambient);
                        audit_log_cap(ab, "pp", &axs->new_pcap.permitted);
                        audit_log_cap(ab, "pi", &axs->new_pcap.inheritable);
                        audit_log_cap(ab, "pe", &axs->new_pcap.effective);
                        audit_log_cap(ab, "pa", &axs->new_pcap.ambient);
                        audit_log_format(ab, " frootid=%d",
                                         from_kuid(&init_user_ns,
                                                   axs->fcap.rootid));
                        break; }

                }
                audit_log_end(ab);
        }

        if (context->type)
                show_special(context, &call_panic);

        if (context->fds[0] >= 0) {
                ab = audit_log_start(context, GFP_KERNEL, AUDIT_FD_PAIR);
                if (ab) {
                        audit_log_format(ab, "fd0=%d fd1=%d",
                                        context->fds[0], context->fds[1]);
                        audit_log_end(ab);
                }
        }

        if (context->sockaddr_len) {
                ab = audit_log_start(context, GFP_KERNEL, AUDIT_SOCKADDR);
                if (ab) {
                        audit_log_format(ab, "saddr=");
                        audit_log_n_hex(ab, (void *)context->sockaddr,
                                        context->sockaddr_len);
                        audit_log_end(ab);
                }
        }

        for (aux = context->aux_pids; aux; aux = aux->next) {
                struct audit_aux_data_pids *axs = (void *)aux;

                for (i = 0; i < axs->pid_count; i++)
                        if (audit_log_pid_context(context, axs->target_pid[i],
                                                  axs->target_auid[i],
                                                  axs->target_uid[i],
                                                  axs->target_sessionid[i],
                                                  &axs->target_ref[i],
                                                  axs->target_comm[i]))
                                call_panic = 1;
        }

        if (context->target_pid &&
            audit_log_pid_context(context, context->target_pid,
                                  context->target_auid, context->target_uid,
                                  context->target_sessionid,
                                  &context->target_ref,
                                  context->target_comm))
                call_panic = 1;

        if (context->pwd.dentry && context->pwd.mnt) {
                ab = audit_log_start(context, GFP_KERNEL, AUDIT_CWD);
                if (ab) {
                        audit_log_d_path(ab, "cwd=", &context->pwd);
                        audit_log_end(ab);
                }
        }

        i = 0;
        list_for_each_entry(n, &context->names_list, list) {
                if (n->hidden)
                        continue;
                audit_log_name(context, n, NULL, i++, &call_panic);
        }

        if (context->context == AUDIT_CTX_SYSCALL)
                audit_log_proctitle();

        /* Send end of event record to help user space know we are finished */
        ab = audit_log_start(context, GFP_KERNEL, AUDIT_EOE);
        if (ab)
                audit_log_end(ab);
        if (call_panic)
                audit_panic("error in audit_log_exit()");
}

/**
 * __audit_free - free a per-task audit context
 * @tsk: task whose audit context block to free
 *
 * Called from copy_process, do_exit, and the io_uring code
 */
void __audit_free(struct task_struct *tsk)
{
        struct audit_context *context = tsk->audit_context;

        if (!context)
                return;

        /* this may generate CONFIG_CHANGE records */
        if (!list_empty(&context->killed_trees))
                audit_kill_trees(context);

        /* We are called either by do_exit() or the fork() error handling code;
         * in the former case tsk == current and in the latter tsk is a
         * random task_struct that doesn't have any meaningful data we
         * need to log via audit_log_exit().
         */
        if (tsk == current && !context->dummy) {
                context->return_valid = AUDITSC_INVALID;
                context->return_code = 0;
                if (context->context == AUDIT_CTX_SYSCALL) {
                        audit_filter_syscall(tsk, context);
                        audit_filter_inodes(tsk, context);
                        if (context->current_state == AUDIT_STATE_RECORD)
                                audit_log_exit();
                } else if (context->context == AUDIT_CTX_URING) {
                        /* TODO: verify this case is real and valid */
                        audit_filter_uring(tsk, context);
                        audit_filter_inodes(tsk, context);
                        if (context->current_state == AUDIT_STATE_RECORD)
                                audit_log_uring(context);
                }
        }

        audit_set_context(tsk, NULL);
        audit_free_context(context);
}

/**
 * audit_return_fixup - fixup the return codes in the audit_context
 * @ctx: the audit_context
 * @success: true/false value to indicate if the operation succeeded or not
 * @code: operation return code
 *
 * We need to fixup the return code in the audit logs if the actual return
 * codes are later going to be fixed by the arch specific signal handlers.
 */
static void audit_return_fixup(struct audit_context *ctx,
                               int success, long code)
{
        /*
         * This is actually a test for:
         * (rc == ERESTARTSYS ) || (rc == ERESTARTNOINTR) ||
         * (rc == ERESTARTNOHAND) || (rc == ERESTART_RESTARTBLOCK)
         *
         * but is faster than a bunch of ||
         */
        if (unlikely(code <= -ERESTARTSYS) &&
            (code >= -ERESTART_RESTARTBLOCK) &&
            (code != -ENOIOCTLCMD))
                ctx->return_code = -EINTR;
        else
                ctx->return_code  = code;
        ctx->return_valid = (success ? AUDITSC_SUCCESS : AUDITSC_FAILURE);
}

/**
 * __audit_uring_entry - prepare the kernel task's audit context for io_uring
 * @op: the io_uring opcode
 *
 * This is similar to audit_syscall_entry() but is intended for use by io_uring
 * operations.  This function should only ever be called from
 * audit_uring_entry() as we rely on the audit context checking present in that
 * function.
 */
void __audit_uring_entry(u8 op)
{
        struct audit_context *ctx = audit_context();

        if (ctx->state == AUDIT_STATE_DISABLED)
                return;

        /*
         * NOTE: It's possible that we can be called from the process' context
         *       before it returns to userspace, and before audit_syscall_exit()
         *       is called.  In this case there is not much to do, just record
         *       the io_uring details and return.
         */
        ctx->uring_op = op;
        if (ctx->context == AUDIT_CTX_SYSCALL)
                return;

        ctx->dummy = !audit_n_rules;
        if (!ctx->dummy && ctx->state == AUDIT_STATE_BUILD)
                ctx->prio = 0;

        ctx->context = AUDIT_CTX_URING;
        ctx->current_state = ctx->state;
        ktime_get_coarse_real_ts64(&ctx->stamp.ctime);
}

/**
 * __audit_uring_exit - wrap up the kernel task's audit context after io_uring
 * @success: true/false value to indicate if the operation succeeded or not
 * @code: operation return code
 *
 * This is similar to audit_syscall_exit() but is intended for use by io_uring
 * operations.  This function should only ever be called from
 * audit_uring_exit() as we rely on the audit context checking present in that
 * function.
 */
void __audit_uring_exit(int success, long code)
{
        struct audit_context *ctx = audit_context();

        if (ctx->dummy) {
                if (ctx->context != AUDIT_CTX_URING)
                        return;
                goto out;
        }

        audit_return_fixup(ctx, success, code);
        if (ctx->context == AUDIT_CTX_SYSCALL) {
                /*
                 * NOTE: See the note in __audit_uring_entry() about the case
                 *       where we may be called from process context before we
                 *       return to userspace via audit_syscall_exit().  In this
                 *       case we simply emit a URINGOP record and bail, the
                 *       normal syscall exit handling will take care of
                 *       everything else.
                 *       It is also worth mentioning that when we are called,
                 *       the current process creds may differ from the creds
                 *       used during the normal syscall processing; keep that
                 *       in mind if/when we move the record generation code.
                 */

                /*
                 * We need to filter on the syscall info here to decide if we
                 * should emit a URINGOP record.  I know it seems odd but this
                 * solves the problem where users have a filter to block *all*
                 * syscall records in the "exit" filter; we want to preserve
                 * the behavior here.
                 */
                audit_filter_syscall(current, ctx);
                if (ctx->current_state != AUDIT_STATE_RECORD)
                        audit_filter_uring(current, ctx);
                audit_filter_inodes(current, ctx);
                if (ctx->current_state != AUDIT_STATE_RECORD)
                        return;

                audit_log_uring(ctx);
                return;
        }

        /* this may generate CONFIG_CHANGE records */
        if (!list_empty(&ctx->killed_trees))
                audit_kill_trees(ctx);

        /* run through both filters to ensure we set the filterkey properly */
        audit_filter_uring(current, ctx);
        audit_filter_inodes(current, ctx);
        if (ctx->current_state != AUDIT_STATE_RECORD)
                goto out;
        audit_log_exit();

out:
        audit_reset_context(ctx);
}

/**
 * __audit_syscall_entry - fill in an audit record at syscall entry
 * @major: major syscall type (function)
 * @a1: additional syscall register 1
 * @a2: additional syscall register 2
 * @a3: additional syscall register 3
 * @a4: additional syscall register 4
 *
 * Fill in audit context at syscall entry.  This only happens if the
 * audit context was created when the task was created and the state or
 * filters demand the audit context be built.  If the state from the
 * per-task filter or from the per-syscall filter is AUDIT_STATE_RECORD,
 * then the record will be written at syscall exit time (otherwise, it
 * will only be written if another part of the kernel requests that it
 * be written).
 */
void __audit_syscall_entry(int major, unsigned long a1, unsigned long a2,
                           unsigned long a3, unsigned long a4)
{
        struct audit_context *context = audit_context();
        enum audit_state     state;

        if (!audit_enabled || !context)
                return;

        WARN_ON(context->context != AUDIT_CTX_UNUSED);
        WARN_ON(context->name_count);
        if (context->context != AUDIT_CTX_UNUSED || context->name_count) {
                audit_panic("unrecoverable error in audit_syscall_entry()");
                return;
        }

        state = context->state;
        if (state == AUDIT_STATE_DISABLED)
                return;

        context->dummy = !audit_n_rules;
        if (!context->dummy && state == AUDIT_STATE_BUILD) {
                context->prio = 0;
                if (auditd_test_task(current))
                        return;
        }

        context->arch            = syscall_get_arch(current);
        context->major      = major;
        context->argv[0]    = a1;
        context->argv[1]    = a2;
        context->argv[2]    = a3;
        context->argv[3]    = a4;
        context->context = AUDIT_CTX_SYSCALL;
        context->current_state  = state;
        ktime_get_coarse_real_ts64(&context->stamp.ctime);
}

/**
 * __audit_syscall_exit - deallocate audit context after a system call
 * @success: success value of the syscall
 * @return_code: return value of the syscall
 *
 * Tear down after system call.  If the audit context has been marked as
 * auditable (either because of the AUDIT_STATE_RECORD state from
 * filtering, or because some other part of the kernel wrote an audit
 * message), then write out the syscall information.  In call cases,
 * free the names stored from getname().
 */
void __audit_syscall_exit(int success, long return_code)
{
        struct audit_context *context = audit_context();

        if (!context || context->dummy ||
            context->context != AUDIT_CTX_SYSCALL)
                goto out;

        /* this may generate CONFIG_CHANGE records */
        if (!list_empty(&context->killed_trees))
                audit_kill_trees(context);

        audit_return_fixup(context, success, return_code);
        /* run through both filters to ensure we set the filterkey properly */
        audit_filter_syscall(current, context);
        audit_filter_inodes(current, context);
        if (context->current_state != AUDIT_STATE_RECORD)
                goto out;

        audit_log_exit();

out:
        audit_reset_context(context);
}

static inline void handle_one(const struct inode *inode)
{
        struct audit_context *context;
        struct audit_tree_refs *p;
        struct audit_chunk *chunk;
        int count;

        if (likely(!inode->i_fsnotify_marks))
                return;
        context = audit_context();
        p = context->trees;
        count = context->tree_count;
        rcu_read_lock();
        chunk = audit_tree_lookup(inode);
        rcu_read_unlock();
        if (!chunk)
                return;
        if (likely(put_tree_ref(context, chunk)))
                return;
        if (unlikely(!grow_tree_refs(context))) {
                pr_warn("out of memory, audit has lost a tree reference\n");
                audit_set_auditable(context);
                audit_put_chunk(chunk);
                unroll_tree_refs(context, p, count);
                return;
        }
        put_tree_ref(context, chunk);
}

static void handle_path(const struct dentry *dentry)
{
        struct audit_context *context;
        struct audit_tree_refs *p;
        const struct dentry *d, *parent;
        struct audit_chunk *drop;
        unsigned long seq;
        int count;

        context = audit_context();
        p = context->trees;
        count = context->tree_count;
retry:
        drop = NULL;
        d = dentry;
        rcu_read_lock();
        seq = read_seqbegin(&rename_lock);
        for (;;) {
                struct inode *inode = d_backing_inode(d);

                if (inode && unlikely(inode->i_fsnotify_marks)) {
                        struct audit_chunk *chunk;

                        chunk = audit_tree_lookup(inode);
                        if (chunk) {
                                if (unlikely(!put_tree_ref(context, chunk))) {
                                        drop = chunk;
                                        break;
                                }
                        }
                }
                parent = d->d_parent;
                if (parent == d)
                        break;
                d = parent;
        }
        if (unlikely(read_seqretry(&rename_lock, seq) || drop)) {  /* in this order */
                rcu_read_unlock();
                if (!drop) {
                        /* just a race with rename */
                        unroll_tree_refs(context, p, count);
                        goto retry;
                }
                audit_put_chunk(drop);
                if (grow_tree_refs(context)) {
                        /* OK, got more space */
                        unroll_tree_refs(context, p, count);
                        goto retry;
                }
                /* too bad */
                pr_warn("out of memory, audit has lost a tree reference\n");
                unroll_tree_refs(context, p, count);
                audit_set_auditable(context);
                return;
        }
        rcu_read_unlock();
}

static struct audit_names *audit_alloc_name(struct audit_context *context,
                                                unsigned char type)
{
        struct audit_names *aname;

        if (context->name_count < AUDIT_NAMES) {
                aname = &context->preallocated_names[context->name_count];
                memset(aname, 0, sizeof(*aname));
        } else {
                aname = kzalloc_obj(*aname, GFP_NOFS);
                if (!aname)
                        return NULL;
                aname->should_free = true;
        }

        aname->ino = AUDIT_INO_UNSET;
        aname->type = type;
        list_add_tail(&aname->list, &context->names_list);

        context->name_count++;
        if (!context->pwd.dentry)
                get_fs_pwd(current->fs, &context->pwd);
        return aname;
}

/**
 * __audit_getname - add a name to the list
 * @name: name to add
 *
 * Add a name to the list of audit names for this context.
 * Called from fs/namei.c:getname().
 */
void __audit_getname(struct filename *name)
{
        struct audit_context *context = audit_context();
        struct audit_names *n;

        if (context->context == AUDIT_CTX_UNUSED)
                return;

        n = audit_alloc_name(context, AUDIT_TYPE_UNKNOWN);
        if (!n)
                return;

        n->name = name;
        n->name_len = AUDIT_NAME_FULL;
        name->aname = n;
        name->refcnt++;
}

static inline int audit_copy_fcaps(struct audit_names *name,
                                   const struct dentry *dentry)
{
        struct cpu_vfs_cap_data caps;
        int rc;

        if (!dentry)
                return 0;

        rc = get_vfs_caps_from_disk(&nop_mnt_idmap, dentry, &caps);
        if (rc)
                return rc;

        name->fcap.permitted = caps.permitted;
        name->fcap.inheritable = caps.inheritable;
        name->fcap.fE = !!(caps.magic_etc & VFS_CAP_FLAGS_EFFECTIVE);
        name->fcap.rootid = caps.rootid;
        name->fcap_ver = (caps.magic_etc & VFS_CAP_REVISION_MASK) >>
                                VFS_CAP_REVISION_SHIFT;

        return 0;
}

/* Copy inode data into an audit_names. */
static void audit_copy_inode(struct audit_names *name,
                             const struct dentry *dentry,
                             struct inode *inode, unsigned int flags)
{
        name->ino   = inode->i_ino;
        name->dev   = inode->i_sb->s_dev;
        name->mode  = inode->i_mode;
        name->uid   = inode->i_uid;
        name->gid   = inode->i_gid;
        name->rdev  = inode->i_rdev;
        security_inode_getlsmprop(inode, &name->oprop);
        if (flags & AUDIT_INODE_NOEVAL) {
                name->fcap_ver = -1;
                return;
        }
        audit_copy_fcaps(name, dentry);
}

/**
 * __audit_inode - store the inode and device from a lookup
 * @name: name being audited
 * @dentry: dentry being audited
 * @flags: attributes for this particular entry
 */
void __audit_inode(struct filename *name, const struct dentry *dentry,
                   unsigned int flags)
{
        struct audit_context *context = audit_context();
        struct inode *inode = d_backing_inode(dentry);
        struct audit_names *n;
        bool parent = flags & AUDIT_INODE_PARENT;
        struct audit_entry *e;
        struct list_head *list = &audit_filter_list[AUDIT_FILTER_FS];
        int i;

        if (context->context == AUDIT_CTX_UNUSED)
                return;

        rcu_read_lock();
        list_for_each_entry_rcu(e, list, list) {
                for (i = 0; i < e->rule.field_count; i++) {
                        struct audit_field *f = &e->rule.fields[i];

                        if (f->type == AUDIT_FSTYPE
                            && audit_comparator(inode->i_sb->s_magic,
                                                f->op, f->val)
                            && e->rule.action == AUDIT_NEVER) {
                                rcu_read_unlock();
                                return;
                        }
                }
        }
        rcu_read_unlock();

        if (!name)
                goto out_alloc;

        /*
         * If we have a pointer to an audit_names entry already, then we can
         * just use it directly if the type is correct.
         */
        n = name->aname;
        if (n) {
                if (parent) {
                        if (n->type == AUDIT_TYPE_PARENT ||
                            n->type == AUDIT_TYPE_UNKNOWN)
                                goto out;
                } else {
                        if (n->type != AUDIT_TYPE_PARENT)
                                goto out;
                }
        }

        list_for_each_entry_reverse(n, &context->names_list, list) {
                if (n->ino) {
                        /* valid inode number, use that for the comparison */
                        if (n->ino != inode->i_ino ||
                            n->dev != inode->i_sb->s_dev)
                                continue;
                } else if (n->name) {
                        /* inode number has not been set, check the name */
                        if (strcmp(n->name->name, name->name))
                                continue;
                } else
                        /* no inode and no name (?!) ... this is odd ... */
                        continue;

                /* match the correct record type */
                if (parent) {
                        if (n->type == AUDIT_TYPE_PARENT ||
                            n->type == AUDIT_TYPE_UNKNOWN)
                                goto out;
                } else {
                        if (n->type != AUDIT_TYPE_PARENT)
                                goto out;
                }
        }

out_alloc:
        /* unable to find an entry with both a matching name and type */
        n = audit_alloc_name(context, AUDIT_TYPE_UNKNOWN);
        if (!n)
                return;
        if (name) {
                n->name = name;
                name->refcnt++;
        }

out:
        if (parent) {
                n->name_len = n->name ? parent_len(n->name->name) : AUDIT_NAME_FULL;
                n->type = AUDIT_TYPE_PARENT;
                if (flags & AUDIT_INODE_HIDDEN)
                        n->hidden = true;
        } else {
                n->name_len = AUDIT_NAME_FULL;
                n->type = AUDIT_TYPE_NORMAL;
        }
        handle_path(dentry);
        audit_copy_inode(n, dentry, inode, flags & AUDIT_INODE_NOEVAL);
}

void __audit_file(const struct file *file)
{
        __audit_inode(NULL, file->f_path.dentry, 0);
}

/**
 * __audit_inode_child - collect inode info for created/removed objects
 * @parent: inode of dentry parent
 * @dentry: dentry being audited
 * @type:   AUDIT_TYPE_* value that we're looking for
 *
 * For syscalls that create or remove filesystem objects, audit_inode
 * can only collect information for the filesystem object's parent.
 * This call updates the audit context with the child's information.
 * Syscalls that create a new filesystem object must be hooked after
 * the object is created.  Syscalls that remove a filesystem object
 * must be hooked prior, in order to capture the target inode during
 * unsuccessful attempts.
 */
void __audit_inode_child(struct inode *parent,
                         const struct dentry *dentry,
                         const unsigned char type)
{
        struct audit_context *context = audit_context();
        struct inode *inode = d_backing_inode(dentry);
        const struct qstr *dname = &dentry->d_name;
        struct audit_names *n, *found_parent = NULL, *found_child = NULL;
        struct audit_entry *e;
        struct list_head *list = &audit_filter_list[AUDIT_FILTER_FS];
        int i;

        if (context->context == AUDIT_CTX_UNUSED)
                return;

        rcu_read_lock();
        list_for_each_entry_rcu(e, list, list) {
                for (i = 0; i < e->rule.field_count; i++) {
                        struct audit_field *f = &e->rule.fields[i];

                        if (f->type == AUDIT_FSTYPE
                            && audit_comparator(parent->i_sb->s_magic,
                                                f->op, f->val)
                            && e->rule.action == AUDIT_NEVER) {
                                rcu_read_unlock();
                                return;
                        }
                }
        }
        rcu_read_unlock();

        if (inode)
                handle_one(inode);

        list_for_each_entry(n, &context->names_list, list) {
                /* can only match entries that have a name */
                if (!n->name)
                        continue;

                /* look for a parent entry first */
                if (!found_parent &&
                    (n->type == AUDIT_TYPE_PARENT || n->type == AUDIT_TYPE_UNKNOWN) &&
                    (n->ino == parent->i_ino && n->dev == parent->i_sb->s_dev &&
                     !audit_compare_dname_path(dname, n->name->name, n->name_len))) {
                        n->type = AUDIT_TYPE_PARENT;
                        found_parent = n;
                        if (found_child)
                                break;
                        continue;
                }

                /* is there a matching child entry? */
                if (!found_child &&
                    (n->type == type || n->type == AUDIT_TYPE_UNKNOWN) &&
                    (!strcmp(dname->name, n->name->name) ||
                     !audit_compare_dname_path(dname, n->name->name,
                                                found_parent ?
                                                found_parent->name_len :
                                                AUDIT_NAME_FULL))) {
                        if (n->type == AUDIT_TYPE_UNKNOWN)
                                n->type = type;
                        found_child = n;
                        if (found_parent)
                                break;
                }
        }

        if (!found_parent) {
                /* create a new, "anonymous" parent record */
                n = audit_alloc_name(context, AUDIT_TYPE_PARENT);
                if (!n)
                        return;
                audit_copy_inode(n, NULL, parent, 0);
        }

        if (!found_child) {
                found_child = audit_alloc_name(context, type);
                if (!found_child)
                        return;

                /* Re-use the name belonging to the slot for a matching parent
                 * directory. All names for this context are relinquished in
                 * audit_free_names() */
                if (found_parent) {
                        found_child->name = found_parent->name;
                        found_child->name_len = AUDIT_NAME_FULL;
                        found_child->name->refcnt++;
                }
        }

        if (inode)
                audit_copy_inode(found_child, dentry, inode, 0);
        else
                found_child->ino = AUDIT_INO_UNSET;
}
EXPORT_SYMBOL_GPL(__audit_inode_child);

/**
 * auditsc_get_stamp - get local copies of audit_context values
 * @ctx: audit_context for the task
 * @stamp: timestamp to record
 *
 * Also sets the context as auditable.
 */
int auditsc_get_stamp(struct audit_context *ctx, struct audit_stamp *stamp)
{
        if (ctx->context == AUDIT_CTX_UNUSED)
                return 0;
        if (!ctx->stamp.serial)
                ctx->stamp.serial = audit_serial();
        *stamp = ctx->stamp;
        if (!ctx->prio) {
                ctx->prio = 1;
                ctx->current_state = AUDIT_STATE_RECORD;
        }
        return 1;
}

/**
 * __audit_mq_open - record audit data for a POSIX MQ open
 * @oflag: open flag
 * @mode: mode bits
 * @attr: queue attributes
 *
 */
void __audit_mq_open(int oflag, umode_t mode, struct mq_attr *attr)
{
        struct audit_context *context = audit_context();

        if (attr)
                memcpy(&context->mq_open.attr, attr, sizeof(struct mq_attr));
        else
                memset(&context->mq_open.attr, 0, sizeof(struct mq_attr));

        context->mq_open.oflag = oflag;
        context->mq_open.mode = mode;

        context->type = AUDIT_MQ_OPEN;
}

/**
 * __audit_mq_sendrecv - record audit data for a POSIX MQ timed send/receive
 * @mqdes: MQ descriptor
 * @msg_len: Message length
 * @msg_prio: Message priority
 * @abs_timeout: Message timeout in absolute time
 *
 */
void __audit_mq_sendrecv(mqd_t mqdes, size_t msg_len, unsigned int msg_prio,
                        const struct timespec64 *abs_timeout)
{
        struct audit_context *context = audit_context();
        struct timespec64 *p = &context->mq_sendrecv.abs_timeout;

        if (abs_timeout)
                memcpy(p, abs_timeout, sizeof(*p));
        else
                memset(p, 0, sizeof(*p));

        context->mq_sendrecv.mqdes = mqdes;
        context->mq_sendrecv.msg_len = msg_len;
        context->mq_sendrecv.msg_prio = msg_prio;

        context->type = AUDIT_MQ_SENDRECV;
}

/**
 * __audit_mq_notify - record audit data for a POSIX MQ notify
 * @mqdes: MQ descriptor
 * @notification: Notification event
 *
 */

void __audit_mq_notify(mqd_t mqdes, const struct sigevent *notification)
{
        struct audit_context *context = audit_context();

        if (notification)
                context->mq_notify.sigev_signo = notification->sigev_signo;
        else
                context->mq_notify.sigev_signo = 0;

        context->mq_notify.mqdes = mqdes;
        context->type = AUDIT_MQ_NOTIFY;
}

/**
 * __audit_mq_getsetattr - record audit data for a POSIX MQ get/set attribute
 * @mqdes: MQ descriptor
 * @mqstat: MQ flags
 *
 */
void __audit_mq_getsetattr(mqd_t mqdes, struct mq_attr *mqstat)
{
        struct audit_context *context = audit_context();

        context->mq_getsetattr.mqdes = mqdes;
        context->mq_getsetattr.mqstat = *mqstat;
        context->type = AUDIT_MQ_GETSETATTR;
}

/**
 * __audit_ipc_obj - record audit data for ipc object
 * @ipcp: ipc permissions
 *
 */
void __audit_ipc_obj(struct kern_ipc_perm *ipcp)
{
        struct audit_context *context = audit_context();

        context->ipc.uid = ipcp->uid;
        context->ipc.gid = ipcp->gid;
        context->ipc.mode = ipcp->mode;
        context->ipc.has_perm = 0;
        security_ipc_getlsmprop(ipcp, &context->ipc.oprop);
        context->type = AUDIT_IPC;
}

/**
 * __audit_ipc_set_perm - record audit data for new ipc permissions
 * @qbytes: msgq bytes
 * @uid: msgq user id
 * @gid: msgq group id
 * @mode: msgq mode (permissions)
 *
 * Called only after audit_ipc_obj().
 */
void __audit_ipc_set_perm(unsigned long qbytes, uid_t uid, gid_t gid, umode_t mode)
{
        struct audit_context *context = audit_context();

        context->ipc.qbytes = qbytes;
        context->ipc.perm_uid = uid;
        context->ipc.perm_gid = gid;
        context->ipc.perm_mode = mode;
        context->ipc.has_perm = 1;
}

void __audit_bprm(struct linux_binprm *bprm)
{
        struct audit_context *context = audit_context();

        context->type = AUDIT_EXECVE;
        context->execve.argc = bprm->argc;
}


/**
 * __audit_socketcall - record audit data for sys_socketcall
 * @nargs: number of args, which should not be more than AUDITSC_ARGS.
 * @args: args array
 *
 */
int __audit_socketcall(int nargs, unsigned long *args)
{
        struct audit_context *context = audit_context();

        if (nargs <= 0 || nargs > AUDITSC_ARGS || !args)
                return -EINVAL;
        context->type = AUDIT_SOCKETCALL;
        context->socketcall.nargs = nargs;
        memcpy(context->socketcall.args, args, nargs * sizeof(unsigned long));
        return 0;
}

/**
 * __audit_fd_pair - record audit data for pipe and socketpair
 * @fd1: the first file descriptor
 * @fd2: the second file descriptor
 *
 */
void __audit_fd_pair(int fd1, int fd2)
{
        struct audit_context *context = audit_context();

        context->fds[0] = fd1;
        context->fds[1] = fd2;
}

/**
 * __audit_sockaddr - record audit data for sys_bind, sys_connect, sys_sendto
 * @len: data length in user space
 * @a: data address in kernel space
 *
 * Returns 0 for success or NULL context or < 0 on error.
 */
int __audit_sockaddr(int len, void *a)
{
        struct audit_context *context = audit_context();

        if (!context->sockaddr) {
                void *p = kmalloc_obj(struct sockaddr_storage);

                if (!p)
                        return -ENOMEM;
                context->sockaddr = p;
        }

        context->sockaddr_len = len;
        memcpy(context->sockaddr, a, len);
        return 0;
}

void __audit_ptrace(struct task_struct *t)
{
        struct audit_context *context = audit_context();

        context->target_pid = task_tgid_nr(t);
        context->target_auid = audit_get_loginuid(t);
        context->target_uid = task_uid(t);
        context->target_sessionid = audit_get_sessionid(t);
        strscpy(context->target_comm, t->comm);
        security_task_getlsmprop_obj(t, &context->target_ref);
}

/**
 * audit_signal_info_syscall - record signal info for syscalls
 * @t: task being signaled
 *
 * If the audit subsystem is being terminated, record the task (pid)
 * and uid that is doing that.
 */
int audit_signal_info_syscall(struct task_struct *t)
{
        struct audit_aux_data_pids *axp;
        struct audit_context *ctx = audit_context();
        kuid_t t_uid = task_uid(t);

        if (!audit_signals || audit_dummy_context())
                return 0;

        /* optimize the common case by putting first signal recipient directly
         * in audit_context */
        if (!ctx->target_pid) {
                ctx->target_pid = task_tgid_nr(t);
                ctx->target_auid = audit_get_loginuid(t);
                ctx->target_uid = t_uid;
                ctx->target_sessionid = audit_get_sessionid(t);
                strscpy(ctx->target_comm, t->comm);
                security_task_getlsmprop_obj(t, &ctx->target_ref);
                return 0;
        }

        axp = (void *)ctx->aux_pids;
        if (!axp || axp->pid_count == AUDIT_AUX_PIDS) {
                axp = kzalloc_obj(*axp, GFP_ATOMIC);
                if (!axp)
                        return -ENOMEM;

                axp->d.type = AUDIT_OBJ_PID;
                axp->d.next = ctx->aux_pids;
                ctx->aux_pids = (void *)axp;
        }
        if (WARN_ON_ONCE(axp->pid_count >= AUDIT_AUX_PIDS))
                return -EINVAL;

        axp->target_pid[axp->pid_count] = task_tgid_nr(t);
        axp->target_auid[axp->pid_count] = audit_get_loginuid(t);
        axp->target_uid[axp->pid_count] = t_uid;
        axp->target_sessionid[axp->pid_count] = audit_get_sessionid(t);
        security_task_getlsmprop_obj(t, &axp->target_ref[axp->pid_count]);
        strscpy(axp->target_comm[axp->pid_count], t->comm);
        axp->pid_count++;

        return 0;
}

/**
 * __audit_log_bprm_fcaps - store information about a loading bprm and relevant fcaps
 * @bprm: pointer to the bprm being processed
 * @new: the proposed new credentials
 * @old: the old credentials
 *
 * Simply check if the proc already has the caps given by the file and if not
 * store the priv escalation info for later auditing at the end of the syscall
 *
 * -Eric
 */
int __audit_log_bprm_fcaps(struct linux_binprm *bprm,
                           const struct cred *new, const struct cred *old)
{
        struct audit_aux_data_bprm_fcaps *ax;
        struct audit_context *context = audit_context();
        struct cpu_vfs_cap_data vcaps;

        ax = kmalloc_obj(*ax);
        if (!ax)
                return -ENOMEM;

        ax->d.type = AUDIT_BPRM_FCAPS;
        ax->d.next = context->aux;
        context->aux = (void *)ax;

        get_vfs_caps_from_disk(&nop_mnt_idmap,
                               bprm->file->f_path.dentry, &vcaps);

        ax->fcap.permitted = vcaps.permitted;
        ax->fcap.inheritable = vcaps.inheritable;
        ax->fcap.fE = !!(vcaps.magic_etc & VFS_CAP_FLAGS_EFFECTIVE);
        ax->fcap.rootid = vcaps.rootid;
        ax->fcap_ver = (vcaps.magic_etc & VFS_CAP_REVISION_MASK) >> VFS_CAP_REVISION_SHIFT;

        ax->old_pcap.permitted   = old->cap_permitted;
        ax->old_pcap.inheritable = old->cap_inheritable;
        ax->old_pcap.effective   = old->cap_effective;
        ax->old_pcap.ambient     = old->cap_ambient;

        ax->new_pcap.permitted   = new->cap_permitted;
        ax->new_pcap.inheritable = new->cap_inheritable;
        ax->new_pcap.effective   = new->cap_effective;
        ax->new_pcap.ambient     = new->cap_ambient;
        return 0;
}

/**
 * __audit_log_capset - store information about the arguments to the capset syscall
 * @new: the new credentials
 * @old: the old (current) credentials
 *
 * Record the arguments userspace sent to sys_capset for later printing by the
 * audit system if applicable
 */
void __audit_log_capset(const struct cred *new, const struct cred *old)
{
        struct audit_context *context = audit_context();

        context->capset.pid = task_tgid_nr(current);
        context->capset.cap.effective   = new->cap_effective;
        context->capset.cap.inheritable = new->cap_inheritable;
        context->capset.cap.permitted   = new->cap_permitted;
        context->capset.cap.ambient     = new->cap_ambient;
        context->type = AUDIT_CAPSET;
}

void __audit_mmap_fd(int fd, int flags)
{
        struct audit_context *context = audit_context();

        context->mmap.fd = fd;
        context->mmap.flags = flags;
        context->type = AUDIT_MMAP;
}

void __audit_openat2_how(struct open_how *how)
{
        struct audit_context *context = audit_context();

        context->openat2.flags = how->flags;
        context->openat2.mode = how->mode;
        context->openat2.resolve = how->resolve;
        context->type = AUDIT_OPENAT2;
}

void __audit_log_kern_module(const char *name)
{
        struct audit_context *context = audit_context();

        context->module.name = kstrdup(name, GFP_KERNEL);
        if (!context->module.name)
                audit_log_lost("out of memory in __audit_log_kern_module");
        context->type = AUDIT_KERN_MODULE;
}

void __audit_fanotify(u32 response, struct fanotify_response_info_audit_rule *friar)
{
        /* {subj,obj}_trust values are {0,1,2}: no,yes,unknown */
        switch (friar->hdr.type) {
        case FAN_RESPONSE_INFO_NONE:
                audit_log(audit_context(), GFP_KERNEL, AUDIT_FANOTIFY,
                          "resp=%u fan_type=%u fan_info=0 subj_trust=2 obj_trust=2",
                          response, FAN_RESPONSE_INFO_NONE);
                break;
        case FAN_RESPONSE_INFO_AUDIT_RULE:
                audit_log(audit_context(), GFP_KERNEL, AUDIT_FANOTIFY,
                          "resp=%u fan_type=%u fan_info=%X subj_trust=%u obj_trust=%u",
                          response, friar->hdr.type, friar->rule_number,
                          friar->subj_trust, friar->obj_trust);
        }
}

void __audit_tk_injoffset(struct timespec64 offset)
{
        struct audit_context *context = audit_context();

        /* only set type if not already set by NTP */
        if (!context->type)
                context->type = AUDIT_TIME_INJOFFSET;
        memcpy(&context->time.tk_injoffset, &offset, sizeof(offset));
}

void __audit_ntp_log(const struct audit_ntp_data *ad)
{
        struct audit_context *context = audit_context();
        int type;

        for (type = 0; type < AUDIT_NTP_NVALS; type++)
                if (ad->vals[type].newval != ad->vals[type].oldval) {
                        /* unconditionally set type, overwriting TK */
                        context->type = AUDIT_TIME_ADJNTPVAL;
                        memcpy(&context->time.ntp_data, ad, sizeof(*ad));
                        break;
                }
}

void __audit_log_nfcfg(const char *name, u8 af, unsigned int nentries,
                       enum audit_nfcfgop op, gfp_t gfp)
{
        struct audit_buffer *ab;
        char comm[sizeof(current->comm)];

        ab = audit_log_start(audit_context(), gfp, AUDIT_NETFILTER_CFG);
        if (!ab)
                return;
        audit_log_format(ab, "table=%s family=%u entries=%u op=%s",
                         name, af, nentries, audit_nfcfgs[op].s);

        audit_log_format(ab, " pid=%u", task_tgid_nr(current));
        audit_log_task_context(ab); /* subj= */
        audit_log_format(ab, " comm=");
        audit_log_untrustedstring(ab, get_task_comm(comm, current));
        audit_log_end(ab);
}
EXPORT_SYMBOL_GPL(__audit_log_nfcfg);

static void audit_log_task(struct audit_buffer *ab)
{
        kuid_t auid, uid;
        kgid_t gid;
        unsigned int sessionid;
        char comm[sizeof(current->comm)];

        auid = audit_get_loginuid(current);
        sessionid = audit_get_sessionid(current);
        current_uid_gid(&uid, &gid);

        audit_log_format(ab, "auid=%u uid=%u gid=%u ses=%u",
                         from_kuid(&init_user_ns, auid),
                         from_kuid(&init_user_ns, uid),
                         from_kgid(&init_user_ns, gid),
                         sessionid);
        audit_log_task_context(ab);
        audit_log_format(ab, " pid=%d comm=", task_tgid_nr(current));
        audit_log_untrustedstring(ab, get_task_comm(comm, current));
        audit_log_d_path_exe(ab, current->mm);
}

/**
 * audit_core_dumps - record information about processes that end abnormally
 * @signr: signal value
 *
 * If a process ends with a core dump, something fishy is going on and we
 * should record the event for investigation.
 */
void audit_core_dumps(long signr)
{
        struct audit_buffer *ab;

        if (!audit_enabled)
                return;

        if (signr == SIGQUIT)        /* don't care for those */
                return;

        ab = audit_log_start(audit_context(), GFP_KERNEL, AUDIT_ANOM_ABEND);
        if (unlikely(!ab))
                return;
        audit_log_task(ab);
        audit_log_format(ab, " sig=%ld res=1", signr);
        audit_log_end(ab);
}

/**
 * audit_seccomp - record information about a seccomp action
 * @syscall: syscall number
 * @signr: signal value
 * @code: the seccomp action
 *
 * Record the information associated with a seccomp action. Event filtering for
 * seccomp actions that are not to be logged is done in seccomp_log().
 * Therefore, this function forces auditing independent of the audit_enabled
 * and dummy context state because seccomp actions should be logged even when
 * audit is not in use.
 */
void audit_seccomp(unsigned long syscall, long signr, int code)
{
        struct audit_buffer *ab;

        ab = audit_log_start(audit_context(), GFP_KERNEL, AUDIT_SECCOMP);
        if (unlikely(!ab))
                return;
        audit_log_task(ab);
        audit_log_format(ab, " sig=%ld arch=%x syscall=%ld compat=%d ip=0x%lx code=0x%x",
                         signr, syscall_get_arch(current), syscall,
                         in_compat_syscall(), KSTK_EIP(current), code);
        audit_log_end(ab);
}

void audit_seccomp_actions_logged(const char *names, const char *old_names,
                                  int res)
{
        struct audit_buffer *ab;

        if (!audit_enabled)
                return;

        ab = audit_log_start(audit_context(), GFP_KERNEL,
                             AUDIT_CONFIG_CHANGE);
        if (unlikely(!ab))
                return;

        audit_log_format(ab,
                         "op=seccomp-logging actions=%s old-actions=%s res=%d",
                         names, old_names, res);
        audit_log_end(ab);
}

struct list_head *audit_killed_trees(void)
{
        struct audit_context *ctx = audit_context();
        if (likely(!ctx || ctx->context == AUDIT_CTX_UNUSED))
                return NULL;
        return &ctx->killed_trees;
}
























   50 
   38 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (c) 2006-2007 Silicon Graphics, Inc.
 * All Rights Reserved.
 */
#ifndef __XFS_FILESTREAM_H__
#define __XFS_FILESTREAM_H__

struct xfs_mount;
struct xfs_inode;
struct xfs_bmalloca;
struct xfs_alloc_arg;

int xfs_filestream_mount(struct xfs_mount *mp);
void xfs_filestream_unmount(struct xfs_mount *mp);
void xfs_filestream_deassociate(struct xfs_inode *ip);
int xfs_filestream_select_ag(struct xfs_bmalloca *ap,
                struct xfs_alloc_arg *args, xfs_extlen_t *blen);

static inline int
xfs_inode_is_filestream(
        struct xfs_inode        *ip)
{
        return xfs_has_filestreams(ip->i_mount) ||
                (ip->i_diflags & XFS_DIFLAG_FILESTREAM);
}

#endif /* __XFS_FILESTREAM_H__ */

































































































    9 










    9 









    9 








    9 
    9 







































    9 





















    9 










    9 

    9 




























    9 

























    8 





    8 

    8 
    8 


























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
/******************************************************************************
 *
 * Module Name: hwvalid - I/O request validation
 *
 * Copyright (C) 2000 - 2026, Intel Corp.
 *
 *****************************************************************************/

#include <acpi/acpi.h>
#include "accommon.h"

#define _COMPONENT          ACPI_HARDWARE
ACPI_MODULE_NAME("hwvalid")

/* Local prototypes */
static acpi_status
acpi_hw_validate_io_request(acpi_io_address address, u32 bit_width);

/*
 * Protected I/O ports. Some ports are always illegal, and some are
 * conditionally illegal. This table must remain ordered by port address.
 *
 * The table is used to implement the Microsoft port access rules that
 * first appeared in Windows XP. Some ports are always illegal, and some
 * ports are only illegal if the BIOS calls _OSI with nothing newer than
 * the specific _OSI strings.
 *
 * This provides ACPICA with the desired port protections and
 * Microsoft compatibility.
 *
 * Description of port entries:
 *  DMA:   DMA controller
 *  PIC0:  Programmable Interrupt Controller (8259A)
 *  PIT1:  System Timer 1
 *  PIT2:  System Timer 2 failsafe
 *  RTC:   Real-time clock
 *  CMOS:  Extended CMOS
 *  DMA1:  DMA 1 page registers
 *  DMA1L: DMA 1 Ch 0 low page
 *  DMA2:  DMA 2 page registers
 *  DMA2L: DMA 2 low page refresh
 *  ARBC:  Arbitration control
 *  SETUP: Reserved system board setup
 *  POS:   POS channel select
 *  PIC1:  Cascaded PIC
 *  IDMA:  ISA DMA
 *  ELCR:  PIC edge/level registers
 *  PCI:   PCI configuration space
 */
static const struct acpi_port_info acpi_protected_ports[] = {
        {"DMA", 0x0000, 0x000F, ACPI_OSI_WIN_XP},
        {"PIC0", 0x0020, 0x0021, ACPI_ALWAYS_ILLEGAL},
        {"PIT1", 0x0040, 0x0043, ACPI_OSI_WIN_XP},
        {"PIT2", 0x0048, 0x004B, ACPI_OSI_WIN_XP},
        {"RTC", 0x0070, 0x0071, ACPI_OSI_WIN_XP},
        {"CMOS", 0x0074, 0x0076, ACPI_OSI_WIN_XP},
        {"DMA1", 0x0081, 0x0083, ACPI_OSI_WIN_XP},
        {"DMA1L", 0x0087, 0x0087, ACPI_OSI_WIN_XP},
        {"DMA2", 0x0089, 0x008B, ACPI_OSI_WIN_XP},
        {"DMA2L", 0x008F, 0x008F, ACPI_OSI_WIN_XP},
        {"ARBC", 0x0090, 0x0091, ACPI_OSI_WIN_XP},
        {"SETUP", 0x0093, 0x0094, ACPI_OSI_WIN_XP},
        {"POS", 0x0096, 0x0097, ACPI_OSI_WIN_XP},
        {"PIC1", 0x00A0, 0x00A1, ACPI_ALWAYS_ILLEGAL},
        {"IDMA", 0x00C0, 0x00DF, ACPI_OSI_WIN_XP},
        {"ELCR", 0x04D0, 0x04D1, ACPI_ALWAYS_ILLEGAL},
        {"PCI", 0x0CF8, 0x0CFF, ACPI_OSI_WIN_XP}
};

#define ACPI_PORT_INFO_ENTRIES      ACPI_ARRAY_LENGTH (acpi_protected_ports)

/******************************************************************************
 *
 * FUNCTION:    acpi_hw_validate_io_request
 *
 * PARAMETERS:  Address             Address of I/O port/register
 *              bit_width           Number of bits (8,16,32)
 *
 * RETURN:      Status
 *
 * DESCRIPTION: Validates an I/O request (address/length). Certain ports are
 *              always illegal and some ports are only illegal depending on
 *              the requests the BIOS AML code makes to the predefined
 *              _OSI method.
 *
 ******************************************************************************/

static acpi_status
acpi_hw_validate_io_request(acpi_io_address address, u32 bit_width)
{
        u32 i;
        u32 byte_width;
        acpi_io_address last_address;
        const struct acpi_port_info *port_info;

        ACPI_FUNCTION_TRACE(hw_validate_io_request);

        /* Supported widths are 8/16/32 */

        if ((bit_width != 8) && (bit_width != 16) && (bit_width != 32)) {
                ACPI_ERROR((AE_INFO,
                            "Bad BitWidth parameter: %8.8X", bit_width));
                return_ACPI_STATUS(AE_BAD_PARAMETER);
        }

        port_info = acpi_protected_ports;
        byte_width = ACPI_DIV_8(bit_width);
        last_address = address + byte_width - 1;

        ACPI_DEBUG_PRINT((ACPI_DB_IO,
                          "Address %8.8X%8.8X LastAddress %8.8X%8.8X Length %X",
                          ACPI_FORMAT_UINT64(address),
                          ACPI_FORMAT_UINT64(last_address), byte_width));

        /* Maximum 16-bit address in I/O space */

        if (last_address > ACPI_UINT16_MAX) {
                ACPI_ERROR((AE_INFO,
                            "Illegal I/O port address/length above 64K: %8.8X%8.8X/0x%X",
                            ACPI_FORMAT_UINT64(address), byte_width));
                return_ACPI_STATUS(AE_LIMIT);
        }

        /* Exit if requested address is not within the protected port table */

        if (address > acpi_protected_ports[ACPI_PORT_INFO_ENTRIES - 1].end) {
                return_ACPI_STATUS(AE_OK);
        }

        /* Check request against the list of protected I/O ports */

        for (i = 0; i < ACPI_PORT_INFO_ENTRIES; i++, port_info++) {
                /*
                 * Check if the requested address range will write to a reserved
                 * port. There are four cases to consider:
                 *
                 * 1) Address range is contained completely in the port address range
                 * 2) Address range overlaps port range at the port range start
                 * 3) Address range overlaps port range at the port range end
                 * 4) Address range completely encompasses the port range
                 */
                if ((address <= port_info->end)
                    && (last_address >= port_info->start)) {

                        /* Port illegality may depend on the _OSI calls made by the BIOS */

                        if (port_info->osi_dependency == ACPI_ALWAYS_ILLEGAL ||
                            acpi_gbl_osi_data == port_info->osi_dependency) {
                                ACPI_DEBUG_PRINT((ACPI_DB_VALUES,
                                                  "Denied AML access to port 0x%8.8X%8.8X/%X (%s 0x%.4X-0x%.4X)\n",
                                                  ACPI_FORMAT_UINT64(address),
                                                  byte_width, port_info->name,
                                                  port_info->start,
                                                  port_info->end));

                                return_ACPI_STATUS(AE_AML_ILLEGAL_ADDRESS);
                        }
                }

                /* Finished if address range ends before the end of this port */

                if (last_address <= port_info->end) {
                        break;
                }
        }

        return_ACPI_STATUS(AE_OK);
}

/******************************************************************************
 *
 * FUNCTION:    acpi_hw_read_port
 *
 * PARAMETERS:  Address             Address of I/O port/register to read
 *              Value               Where value (data) is returned
 *              Width               Number of bits
 *
 * RETURN:      Status and value read from port
 *
 * DESCRIPTION: Read data from an I/O port or register. This is a front-end
 *              to acpi_os_read_port that performs validation on both the port
 *              address and the length.
 *
 *****************************************************************************/

acpi_status acpi_hw_read_port(acpi_io_address address, u32 *value, u32 width)
{
        acpi_status status;
        u32 one_byte;
        u32 i;

        /* Truncate address to 16 bits if requested */

        if (acpi_gbl_truncate_io_addresses) {
                address &= ACPI_UINT16_MAX;
        }

        /* Validate the entire request and perform the I/O */

        status = acpi_hw_validate_io_request(address, width);
        if (ACPI_SUCCESS(status)) {
                status = acpi_os_read_port(address, value, width);
                return (status);
        }

        if (status != AE_AML_ILLEGAL_ADDRESS) {
                return (status);
        }

        /*
         * There has been a protection violation within the re